aboutsummaryrefslogtreecommitdiff
path: root/driver
diff options
context:
space:
mode:
authorDrew Richardson <drew.richardson@arm.com>2011-09-30 12:00:00 -0700
committerDrew Richardson <drew.richardson@arm.com>2014-12-19 15:16:17 -0800
commitc5f78695067d4614bcb494d5f54bde4405c43609 (patch)
tree5413ee61289c45be1917e93852d491aa1f778087 /driver
parent73158f9d05b7087bb6ff895e820017af3c94a1a6 (diff)
gator: Version 5.75.7
Signed-off-by: Drew Richardson <drew.richardson@arm.com>
Diffstat (limited to 'driver')
-rw-r--r--driver/Makefile5
-rw-r--r--driver/gator.h12
-rw-r--r--driver/gator_annotate.c28
-rw-r--r--driver/gator_backtrace.c6
-rw-r--r--driver/gator_cookies.c40
-rw-r--r--driver/gator_ebs.c160
-rw-r--r--driver/gator_events_armv6.c40
-rw-r--r--driver/gator_events_armv7.c186
-rw-r--r--driver/gator_events_armv7.h38
-rw-r--r--driver/gator_events_l2c-310.c182
-rw-r--r--driver/gator_events_meminfo.c11
-rw-r--r--driver/gator_events_pl310.c176
-rw-r--r--driver/gator_events_scorpion.c40
-rw-r--r--driver/gator_main.c528
-rw-r--r--driver/gator_pack.c262
-rw-r--r--driver/gator_trace_sched.c228
16 files changed, 1174 insertions, 768 deletions
diff --git a/driver/Makefile b/driver/Makefile
index b3680e1..de59957 100644
--- a/driver/Makefile
+++ b/driver/Makefile
@@ -17,7 +17,8 @@ endif
gator-$(CONFIG_ARM) += gator_events_armv6.o \
gator_events_armv7.o \
- gator_events_pl310.o
+ gator_events_l2c-310.o \
+ gator_events_scorpion.o
$(obj)/gator_main.o: gator_events.h
@@ -40,6 +41,6 @@ all:
$(error)
clean:
- rm -f *.o modules.order Module.symvers gator.ko gator.mod.c
+ rm -f *.o .*.cmd gator_events.h modules.order Module.symvers gator.ko gator.mod.c
endif
diff --git a/driver/gator.h b/driver/gator.h
index 9d802a3..724ae19 100644
--- a/driver/gator.h
+++ b/driver/gator.h
@@ -14,6 +14,16 @@
#include <linux/mm.h>
#include <linux/list.h>
+// cpu ids
+#define ARM1136 0xb36
+#define ARM1156 0xb56
+#define ARM1176 0xb76
+#define ARM11MPCORE 0xb02
+#define CORTEX_A5 0xc05
+#define CORTEX_A8 0xc08
+#define CORTEX_A9 0xc09
+#define CORTEX_A15 0xc0f
+
/******************************************************************************
* Filesystem
******************************************************************************/
@@ -62,6 +72,7 @@ struct gator_interface {
void (*online)(void);
void (*offline)(void);
int (*read)(int **buffer);
+ int (*read64)(long long **buffer);
struct list_head list;
};
@@ -75,5 +86,4 @@ extern u32 gator_cpuid(void);
extern unsigned long gator_net_traffic;
-
#endif // GATOR_H_
diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c
index d8d86af..e096d15 100644
--- a/driver/gator_annotate.c
+++ b/driver/gator_annotate.c
@@ -22,6 +22,7 @@ static char *annotateBuf0;
static char *annotateBuf1;
static int annotatePos;
static int annotateSel;
+static bool collect_annotations = false;
static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
{
@@ -45,16 +46,13 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t
if (retval == 0) {
// synchronize shared variables annotateBuf and annotatePos
spin_lock(&annotate_lock);
- if (annotateBuf) {
+ if (collect_annotations && annotateBuf) {
uint32_t tid = current->pid;
- uint32_t tick = gator_master_tick;
uint64_t time = gator_get_time();
uint32_t cpuid = smp_processor_id();
int pos = annotatePos;
pos += gator_write_packed_int(&annotateBuf[pos], tid);
- pos += gator_write_packed_int(&annotateBuf[pos], tick);
- pos += gator_write_packed_int(&annotateBuf[pos], time);
- pos += gator_write_packed_int(&annotateBuf[pos], time >> 32);
+ pos += gator_write_packed_int64(&annotateBuf[pos], time);
pos += gator_write_packed_int(&annotateBuf[pos], cpuid);
pos += gator_write_packed_int(&annotateBuf[pos], size);
memcpy(&annotateBuf[pos], tempBuffer, size);
@@ -81,12 +79,9 @@ static int annotate_release(struct inode *inode, struct file *file)
spin_lock(&annotate_lock);
if (annotateBuf) {
uint32_t tid = current->pid;
- uint32_t tick = gator_master_tick;
int pos = annotatePos;
pos += gator_write_packed_int(&annotateBuf[pos], tid);
- pos += gator_write_packed_int(&annotateBuf[pos], tick);
- pos += gator_write_packed_int(&annotateBuf[pos], 0); // time
- pos += gator_write_packed_int(&annotateBuf[pos], 0); // time
+ pos += gator_write_packed_int64(&annotateBuf[pos], 0); // time
pos += gator_write_packed_int(&annotateBuf[pos], 0); // cpuid
pos += gator_write_packed_int(&annotateBuf[pos], 0); // size
annotatePos = pos;
@@ -118,13 +113,21 @@ static int gator_annotate_init(void)
static int gator_annotate_start(void)
{
- annotatePos = annotateSel = 0;
+ annotateSel = 0;
+ annotatePos = 1;
annotateBuf = annotateBuf0;
+ annotateBuf[0] = FRAME_ANNOTATE;
+ collect_annotations = true;
return 0;
}
static void gator_annotate_stop(void)
{
+ collect_annotations = false;
+}
+
+static void gator_annotate_shutdown(void)
+{
spin_lock(&annotate_lock);
annotateBuf = NULL;
spin_unlock(&annotate_lock);
@@ -141,7 +144,7 @@ static void gator_annotate_exit(void)
static int gator_annotate_ready(void)
{
- return annotatePos && annotateBuf;
+ return annotatePos > 1 && annotateBuf;
}
static int gator_annotate_read(char **buffer)
@@ -158,8 +161,9 @@ static int gator_annotate_read(char **buffer)
spin_lock(&annotate_lock);
len = annotatePos;
- annotatePos = 0;
annotateBuf = annotateSel ? annotateBuf1 : annotateBuf0;
+ annotateBuf[0] = FRAME_ANNOTATE;
+ annotatePos = 1;
spin_unlock(&annotate_lock);
return len;
diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c
index 628a18f..b5b0e63 100644
--- a/driver/gator_backtrace.c
+++ b/driver/gator_backtrace.c
@@ -15,7 +15,7 @@ struct frame_tail_eabi {
unsigned long lr;
};
-static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned int depth)
+static void arm_backtrace_eabi(int cpu, int buftype, struct pt_regs * const regs, unsigned int depth)
{
#if defined(__arm__)
struct frame_tail_eabi *tail;
@@ -31,7 +31,7 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned in
}
/* entry preamble may not have executed */
- gator_add_trace(cpu, lr);
+ gator_add_trace(cpu, buftype, lr);
/* check tail is valid */
if (fp == 0) {
@@ -49,7 +49,7 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned in
ptrtail = &buftail;
lr = ptrtail[0].lr;
- gator_add_trace(cpu, lr);
+ gator_add_trace(cpu, buftype, lr);
/* frame pointers should progress back up the stack, towards higher addresses */
next = (struct frame_tail_eabi *)(lr - 4);
diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c
index 64be841..a646fb2 100644
--- a/driver/gator_cookies.c
+++ b/driver/gator_cookies.c
@@ -22,7 +22,7 @@ static DEFINE_PER_CPU(int, translate_buffer_read);
static DEFINE_PER_CPU(int, translate_buffer_write);
static DEFINE_PER_CPU(unsigned int *, translate_buffer);
-static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod);
+static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt);
static void wq_cookie_handler(struct work_struct *unused);
DECLARE_WORK(cookie_work, wq_cookie_handler);
@@ -96,7 +96,7 @@ static void cookiemap_add(uint64_t key, uint32_t value) {
for (x = MAX_COLLISIONS-1; x > 0; x--) {
keys[x] = keys[x-1];
- values[x] = keys[x-1];
+ values[x] = values[x-1];
}
keys[0] = key;
values[0] = value;
@@ -126,12 +126,12 @@ static void wq_cookie_handler(struct work_struct *unused)
while (per_cpu(translate_buffer_read, cpu) != commit) {
task = (struct task_struct *)translate_buffer_read_int(cpu);
vma = (struct vm_area_struct *)translate_buffer_read_int(cpu);
- cookie = get_cookie(cpu, task, vma, NULL);
+ cookie = get_cookie(cpu, TIMER_BUF, task, vma, NULL, false);
}
}
// Retrieve full name from proc/pid/cmdline for java processes on Android
-static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma)
+static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma, bool in_interrupt)
{
void *maddr;
unsigned int len;
@@ -143,7 +143,9 @@ static int translate_app_process(char** text, int cpu, struct task_struct * task
char * buf = per_cpu(translate_text, cpu);
// Push work into a work queue if in atomic context as the kernel functions below might sleep
- if (in_irq()) {
+ // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
+ // inconsistent during a context switch between android/linux versions
+ if (in_interrupt) {
// Check if already in buffer
ptr = per_cpu(translate_buffer_read, cpu);
while (ptr != per_cpu(translate_buffer_write, cpu)) {
@@ -205,7 +207,7 @@ out:
return retval;
}
-static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod)
+static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt)
{
unsigned long flags, cookie;
struct path *path;
@@ -235,26 +237,26 @@ static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_a
}
if (strcmp(text, "app_process") == 0 && !mod) {
- if (!translate_app_process(&text, cpu, task, vma))
+ if (!translate_app_process(&text, cpu, task, vma, in_interrupt))
return INVALID_COOKIE;
}
- // Can be called from interrupt handler or from work queue
+ // Can be called from interrupt handler or from work queue or from scheduler trace
local_irq_save(flags);
cookie = per_cpu(cookie_next_key, cpu)+=nr_cpu_ids;
cookiemap_add(key, cookie);
- gator_buffer_write_packed_int(cpu, PROTOCOL_COOKIE);
- gator_buffer_write_packed_int(cpu, cookie);
- gator_buffer_write_string(cpu, text);
+ gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE);
+ gator_buffer_write_packed_int(cpu, buftype, cookie);
+ gator_buffer_write_string(cpu, buftype, text);
local_irq_restore(flags);
return cookie;
}
-static int get_exec_cookie(int cpu, struct task_struct *task)
+static int get_exec_cookie(int cpu, int buftype, struct task_struct *task)
{
unsigned long cookie = NO_COOKIE;
struct mm_struct *mm = task->mm;
@@ -268,14 +270,14 @@ static int get_exec_cookie(int cpu, struct task_struct *task)
continue;
if (!(vma->vm_flags & VM_EXECUTABLE))
continue;
- cookie = get_cookie(cpu, task, vma, NULL);
+ cookie = get_cookie(cpu, buftype, task, vma, NULL, true);
break;
}
return cookie;
}
-static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
+static unsigned long get_address_cookie(int cpu, int buftype, struct task_struct *task, unsigned long addr, off_t *offset)
{
unsigned long cookie = NO_COOKIE;
struct mm_struct *mm = task->mm;
@@ -289,7 +291,7 @@ static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsig
continue;
if (vma->vm_file) {
- cookie = get_cookie(cpu, task, vma, NULL);
+ cookie = get_cookie(cpu, buftype, task, vma, NULL, true);
*offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
} else {
/* must be an anonymous map */
@@ -318,10 +320,18 @@ static int cookies_initialize(void)
size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
per_cpu(cookie_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL);
+ if (!per_cpu(cookie_keys, cpu)) {
+ err = -ENOMEM;
+ goto cookie_setup_error;
+ }
memset(per_cpu(cookie_keys, cpu), 0, size);
size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL);
+ if (!per_cpu(cookie_values, cpu)) {
+ err = -ENOMEM;
+ goto cookie_setup_error;
+ }
memset(per_cpu(cookie_values, cpu), 0, size);
per_cpu(translate_buffer, cpu) = (unsigned int *)kmalloc(translate_buffer_size, GFP_KERNEL);
diff --git a/driver/gator_ebs.c b/driver/gator_ebs.c
new file mode 100644
index 0000000..9b55347
--- /dev/null
+++ b/driver/gator_ebs.c
@@ -0,0 +1,160 @@
+/**
+ * Copyright (C) ARM Limited 2011. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+/******************************************************************************
+ * event based sampling handling
+ ******************************************************************************/
+
+#if defined (__arm__)
+#include "gator_events_armv7.h"
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+
+#if LINUX_PMU_SUPPORT
+#include <asm/pmu.h>
+
+static struct platform_device *pmu_device;
+
+static irqreturn_t armv7_pmnc_interrupt(int irq, void *arg)
+{
+ unsigned int cnt, cpu = smp_processor_id(), buftype = EVENT_BUF;
+ struct pt_regs * const regs = get_irq_regs();
+ u32 flags;
+
+ // Stop irq generation
+ armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E);
+
+ // Get and reset overflow status flags
+ flags = armv7_pmnc_reset_interrupt();
+
+ // Counters header
+ gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
+ gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
+
+ // Cycle counter
+ if (flags & (1 << 31)) {
+ int value = armv7_ccnt_read(pmnc_count[CCNT]); // overrun
+ gator_buffer_write_packed_int(cpu, buftype, 2); // length
+ gator_buffer_write_packed_int(cpu, buftype, pmnc_key[CCNT]); // key
+ gator_buffer_write_packed_int(cpu, buftype, value); // value
+ }
+
+ // PMNC counters
+ for (cnt = CNT0; cnt < CNTMAX; cnt++) {
+ if (flags & (1 << (cnt - CNT0))) {
+ int value = armv7_cntn_read(cnt, pmnc_count[cnt]); // overrun
+ gator_buffer_write_packed_int(cpu, buftype, 2); // length
+ gator_buffer_write_packed_int(cpu, buftype, pmnc_key[cnt]); // key
+ gator_buffer_write_packed_int(cpu, buftype, value); // value
+ }
+ }
+
+ // End Counters, length of zero
+ gator_buffer_write_packed_int(cpu, buftype, 0);
+
+ // Output backtrace
+ gator_add_sample(cpu, buftype, regs);
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ event_buffer_check(cpu);
+
+ // Allow irq generation
+ armv7_pmnc_write(armv7_pmnc_read() | PMNC_E);
+
+ return IRQ_HANDLED;
+}
+#endif
+
+static int gator_event_sampling_start(void)
+{
+ int cnt;
+
+ event_based_sampling = false;
+ for (cnt = CCNT; cnt < CNTMAX; cnt++) {
+ if (pmnc_count[cnt] > 0) {
+ event_based_sampling = true;
+ break;
+ }
+ }
+
+#if LINUX_PMU_SUPPORT
+ pmu_device = reserve_pmu(ARM_PMU_DEVICE_CPU);
+ if (IS_ERR(pmu_device) && (unsigned int)pmu_device != -ENODEV) {
+ pr_err("gator: unable to reserve the pmu\n");
+ return -1;
+ }
+
+ if (event_based_sampling) {
+ int irq, i;
+
+ if (IS_ERR(pmu_device)) {
+ pr_err("gator: event based sampling is not supported as the kernel function reserve_pmu() failed");
+ return -1;
+ }
+
+ init_pmu(ARM_PMU_DEVICE_CPU);
+ if (pmu_device->num_resources == 0) {
+ pr_err("gator: no irqs for PMUs defined\n");
+ release_pmu(pmu_device);
+ pmu_device = NULL;
+ return -1;
+ }
+
+ for (i = 0; i < pmu_device->num_resources; ++i) {
+ irq = platform_get_irq(pmu_device, i);
+ if (irq < 0)
+ continue;
+
+ if (request_irq(irq, armv7_pmnc_interrupt, IRQF_DISABLED | IRQF_NOBALANCING, "armpmu", NULL)) {
+ pr_err("gator: unable to request IRQ%d for ARM perf counters\n", irq);
+
+ // clean up and exit
+ for (i = i - 1; i >= 0; --i) {
+ irq = platform_get_irq(pmu_device, i);
+ if (irq >= 0)
+ free_irq(irq, NULL);
+ }
+ release_pmu(pmu_device);
+ pmu_device = NULL;
+ return -1;
+ }
+ }
+ }
+#else
+ if (event_based_sampling) {
+ pr_err("gator: event based sampling only supported in kernel versions 2.6.35 and higher and CONFIG_CPU_HAS_PMU=y\n");
+ return -1;
+ }
+#endif
+
+ return 0;
+}
+
+static void gator_event_sampling_stop(void)
+{
+#if LINUX_PMU_SUPPORT
+ if (event_based_sampling) {
+ int i, irq;
+ for (i = pmu_device->num_resources - 1; i >= 0; --i) {
+ irq = platform_get_irq(pmu_device, i);
+ if (irq >= 0)
+ free_irq(irq, NULL);
+ }
+ }
+ if (!IS_ERR(pmu_device))
+ release_pmu(pmu_device);
+ pmu_device = NULL;
+#endif
+}
+
+#else
+static int gator_event_sampling_start(void) {return 0;}
+static void gator_event_sampling_stop(void) {}
+#endif
diff --git a/driver/gator_events_armv6.c b/driver/gator_events_armv6.c
index 7b1d875..170f066 100644
--- a/driver/gator_events_armv6.c
+++ b/driver/gator_events_armv6.c
@@ -8,10 +8,6 @@
#include "gator.h"
-#define ARM1136 0xb36
-#define ARM1156 0xb56
-#define ARM1176 0xb76
-
static const char *pmnc_name;
/*
@@ -29,9 +25,10 @@ static const char *pmnc_name;
#define CCNT 2
#define CNTMAX (CCNT+1)
-static int pmnc_count = 0;
+static int pmnc_counters = 0;
static unsigned long pmnc_enabled[CNTMAX];
static unsigned long pmnc_event[CNTMAX];
+static unsigned long pmnc_count[CNTMAX];
static unsigned long pmnc_key[CNTMAX];
static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
@@ -72,7 +69,7 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root)
struct dentry *dir;
int i;
- pmnc_count = 3;
+ pmnc_counters = 3;
for (i = PMN0; i <= CCNT; i++) {
char buf[40];
@@ -86,10 +83,11 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root)
return -1;
}
gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]);
+ gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]);
+ gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
if (i != CCNT) {
gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]);
}
- gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
}
return 0;
@@ -119,18 +117,14 @@ static void gator_events_armv6_online(void)
event = pmnc_event[cnt] & 255;
- /*
- * Set event (if destined for PMNx counters)
- */
+ // Set event (if destined for PMNx counters)
if (cnt == PMN0) {
pmnc |= event << 20;
} else if (cnt == PMN1) {
pmnc |= event << 12;
}
- /*
- * Reset counter
- */
+ // Reset counter
armv6_pmnc_reset_counter(cnt);
}
armv6_pmnc_write(pmnc | PMCR_E);
@@ -146,6 +140,20 @@ static void gator_events_armv6_offline(void)
}
}
+static int gator_events_armv6_start(void)
+{
+ int cnt;
+
+ for (cnt = CCNT; cnt < CNTMAX; cnt++) {
+ if (pmnc_count[cnt] > 0) {
+ pr_err("gator: event based sampling not supported on ARM v6 architectures\n");
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
static void gator_events_armv6_stop(void)
{
unsigned int cnt;
@@ -153,6 +161,7 @@ static void gator_events_armv6_stop(void)
for (cnt = PMN0; cnt <= CCNT; cnt++) {
pmnc_enabled[cnt] = 0;
pmnc_event[cnt] = 0;
+ pmnc_count[cnt] = 0;
}
}
@@ -193,6 +202,7 @@ static int gator_events_armv6_read(int **buffer)
static struct gator_interface gator_events_armv6_interface = {
.create_files = gator_events_armv6_create_files,
+ .start = gator_events_armv6_start,
.stop = gator_events_armv6_stop,
.online = gator_events_armv6_online,
.offline = gator_events_armv6_offline,
@@ -209,6 +219,9 @@ int gator_events_armv6_init(void)
case ARM1176:
pmnc_name = "ARM11";
break;
+ case ARM11MPCORE:
+ pmnc_name = "ARM11MPCore";
+ break;
default:
return -1;
}
@@ -216,6 +229,7 @@ int gator_events_armv6_init(void)
for (cnt = PMN0; cnt <= CCNT; cnt++) {
pmnc_enabled[cnt] = 0;
pmnc_event[cnt] = 0;
+ pmnc_count[cnt] = 0;
pmnc_key[cnt] = gator_events_get_key();
}
diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c
index 0f0d7ef..58855f8 100644
--- a/driver/gator_events_armv7.c
+++ b/driver/gator_events_armv7.c
@@ -6,129 +6,116 @@
* published by the Free Software Foundation.
*/
-#include "gator.h"
-
-#define CORTEX_A5 0xc05
-#define CORTEX_A8 0xc08
-#define CORTEX_A9 0xc09
-#define CORTEX_A15 0xc0f
-
-static const char *pmnc_name;
-static int pmnc_count;
-
-// Per-CPU PMNC: config reg
-#define PMNC_E (1 << 0) /* Enable all counters */
-#define PMNC_P (1 << 1) /* Reset all counters */
-#define PMNC_C (1 << 2) /* Cycle counter reset */
-#define PMNC_MASK 0x3f /* Mask for writable bits */
+/* Disabling interrupts
+ * Many of the functions below disable interrupts via local_irq_save(). This disabling of interrupts is done to prevent any race conditions
+ * between multiple entities (e.g. hrtimer interrupts and event based interrupts) calling the same functions. As accessing the pmu involves
+ * several steps (disable, select, read, enable), these steps must be performed atomically. Normal synchronization routines cannot be used
+ * as these functions are being called from interrupt context.
+ */
-// ccnt reg
-#define CCNT_REG (1 << 31)
+#include "gator.h"
+#include "gator_events_armv7.h"
-#define CCNT 0
-#define CNT0 1
-#define CNTMAX (6+1)
+const char *pmnc_name;
+int pmnc_counters;
-static unsigned long pmnc_enabled[CNTMAX];
-static unsigned long pmnc_event[CNTMAX];
-static unsigned long pmnc_key[CNTMAX];
+unsigned long pmnc_enabled[CNTMAX];
+unsigned long pmnc_event[CNTMAX];
+unsigned long pmnc_count[CNTMAX];
+unsigned long pmnc_key[CNTMAX];
static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt);
-static inline void armv7_pmnc_write(u32 val)
+inline void armv7_pmnc_write(u32 val)
{
val &= PMNC_MASK;
asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r" (val));
}
-static inline u32 armv7_pmnc_read(void)
+inline u32 armv7_pmnc_read(void)
{
u32 val;
asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
return val;
}
-static inline u32 armv7_ccnt_read(void)
+inline u32 armv7_ccnt_read(u32 reset_value)
{
- u32 zero = 0;
+ unsigned long flags;
+ u32 newval = -reset_value;
u32 den = CCNT_REG;
u32 val;
+ local_irq_save(flags);
asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable
asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); // read
- asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (zero)); // zero
+ asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (newval));// new value
asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable
+ local_irq_restore(flags);
return val;
}
-static inline u32 armv7_cntn_read(unsigned int cnt)
+inline u32 armv7_cntn_read(unsigned int cnt, u32 reset_value)
{
- u32 zero = 0;
+ unsigned long flags;
+ u32 newval = -reset_value;
u32 sel = (cnt - CNT0);
u32 den = 1 << sel;
- u32 val;
+ u32 oldval;
+ local_irq_save(flags);
asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable
asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (sel)); // select
- asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); // read
- asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (zero)); // zero
+ asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (oldval)); // read
+ asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (newval));// new value
asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable
+ local_irq_restore(flags);
- return val;
+ return oldval;
}
-static inline u32 armv7_pmnc_enable_counter(unsigned int cnt)
+static inline void armv7_pmnc_enable_interrupt(unsigned int cnt)
{
- u32 val;
+ u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31);
+ asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val));
+}
- if (cnt >= CNTMAX) {
- pr_err("gator: CPU%u enabling wrong PMNC counter %d\n", smp_processor_id(), cnt);
- return -1;
- }
+static inline void armv7_pmnc_disable_interrupt(unsigned int cnt)
+{
+ u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31);
+ asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val));
+}
- if (cnt == CCNT)
- val = CCNT_REG;
- else
- val = (1 << (cnt - CNT0));
+inline u32 armv7_pmnc_reset_interrupt()
+{
+ // Get and reset overflow status flags
+ u32 flags;
+ asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (flags));
+ flags &= 0x8000003f;
+ asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (flags));
+ return flags;
+}
+static inline u32 armv7_pmnc_enable_counter(unsigned int cnt)
+{
+ u32 val = cnt ? (1 << (cnt - CNT0)) : CCNT_REG;
asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val));
-
return cnt;
}
static inline u32 armv7_pmnc_disable_counter(unsigned int cnt)
{
- u32 val;
-
- if (cnt >= CNTMAX) {
- pr_err("gator: CPU%u disabling wrong PMNC counter %d\n", smp_processor_id(), cnt);
- return -1;
- }
-
- if (cnt == CCNT)
- val = CCNT_REG;
- else
- val = (1 << (cnt - CNT0));
-
+ u32 val = cnt ? (1 << (cnt - CNT0)) : CCNT_REG;
asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val));
-
return cnt;
}
static inline int armv7_pmnc_select_counter(unsigned int cnt)
{
- u32 val;
-
- if ((cnt == CCNT) || (cnt >= CNTMAX)) {
- pr_err("gator: CPU%u selecting wrong PMNC counter %d\n", smp_processor_id(), cnt);
- return -1;
- }
-
- val = (cnt - CNT0);
+ u32 val = (cnt - CNT0);
asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val));
-
return cnt;
}
@@ -139,37 +126,12 @@ static inline void armv7_pmnc_write_evtsel(unsigned int cnt, u32 val)
}
}
-static void armv7_pmnc_reset_counter(unsigned int cnt)
-{
- u32 val = 0;
-
- if (cnt == CCNT) {
- armv7_pmnc_disable_counter(cnt);
-
- asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (val));
-
- if (pmnc_enabled[cnt] != 0)
- armv7_pmnc_enable_counter(cnt);
-
- } else if (cnt >= CNTMAX) {
- pr_err("gator: CPU%u resetting wrong PMNC counter %d\n", smp_processor_id(), cnt);
- } else {
- armv7_pmnc_disable_counter(cnt);
-
- if (armv7_pmnc_select_counter(cnt) == cnt)
- asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (val));
-
- if (pmnc_enabled[cnt] != 0)
- armv7_pmnc_enable_counter(cnt);
- }
-}
-
static int gator_events_armv7_create_files(struct super_block *sb, struct dentry *root)
{
struct dentry *dir;
int i;
- for (i = 0; i < pmnc_count; i++) {
+ for (i = 0; i < pmnc_counters; i++) {
char buf[40];
if (i == 0) {
snprintf(buf, sizeof buf, "ARM_%s_ccnt", pmnc_name);
@@ -181,10 +143,11 @@ static int gator_events_armv7_create_files(struct super_block *sb, struct dentry
return -1;
}
gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]);
+ gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]);
+ gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
if (i > 0) {
gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]);
}
- gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
}
return 0;
@@ -202,6 +165,9 @@ static void gator_events_armv7_online(void)
// Initialize & Reset PMNC: C bit and P bit
armv7_pmnc_write(PMNC_P | PMNC_C);
+ // Reset overflow flags
+ armv7_pmnc_reset_interrupt();
+
for (cnt = CCNT; cnt < CNTMAX; cnt++) {
unsigned long event;
@@ -219,10 +185,16 @@ static void gator_events_armv7_online(void)
if (cnt != CCNT)
armv7_pmnc_write_evtsel(cnt, event);
+ // Enable/disable interrupt
+ if (pmnc_count[cnt] > 0)
+ armv7_pmnc_enable_interrupt(cnt);
+ else
+ armv7_pmnc_disable_interrupt(cnt);
+
// Reset counter
- armv7_pmnc_reset_counter(cnt);
+ cnt ? armv7_cntn_read(cnt, pmnc_count[cnt]) : armv7_ccnt_read(pmnc_count[cnt]);
- // Enable counter, but do not enable interrupt for this counter
+ // Enable counter
armv7_pmnc_enable_counter(cnt);
}
@@ -242,6 +214,7 @@ static void gator_events_armv7_stop(void)
for (cnt = CCNT; cnt < CNTMAX; cnt++) {
pmnc_enabled[cnt] = 0;
pmnc_event[cnt] = 0;
+ pmnc_count[cnt] = 0;
}
}
@@ -250,16 +223,16 @@ static int gator_events_armv7_read(int **buffer)
int cnt, len = 0;
int cpu = smp_processor_id();
- if (!pmnc_count)
+ if (!pmnc_counters)
return 0;
- for (cnt = 0; cnt < pmnc_count; cnt++) {
- if (pmnc_enabled[cnt]) {
+ for (cnt = 0; cnt < pmnc_counters; cnt++) {
+ if (pmnc_enabled[cnt] && pmnc_count[cnt] == 0) {
int value;
if (cnt == CCNT) {
- value = armv7_ccnt_read();
+ value = armv7_ccnt_read(0);
} else {
- value = armv7_cntn_read(cnt);
+ value = armv7_cntn_read(cnt, 0);
}
if (value != per_cpu(perfPrev, cpu)[cnt]) {
per_cpu(perfPrev, cpu)[cnt] = value;
@@ -291,29 +264,30 @@ int gator_events_armv7_init(void)
switch (gator_cpuid()) {
case CORTEX_A5:
pmnc_name = "Cortex-A5";
- pmnc_count = 2;
+ pmnc_counters = 2;
break;
case CORTEX_A8:
pmnc_name = "Cortex-A8";
- pmnc_count = 4;
+ pmnc_counters = 4;
break;
case CORTEX_A9:
pmnc_name = "Cortex-A9";
- pmnc_count = 6;
+ pmnc_counters = 6;
break;
case CORTEX_A15:
pmnc_name = "Cortex-A15";
- pmnc_count = 6;
+ pmnc_counters = 6;
break;
default:
return -1;
}
- pmnc_count++; // CNT[n] + CCNT
+ pmnc_counters++; // CNT[n] + CCNT
for (cnt = CCNT; cnt < CNTMAX; cnt++) {
pmnc_enabled[cnt] = 0;
pmnc_event[cnt] = 0;
+ pmnc_count[cnt] = 0;
pmnc_key[cnt] = gator_events_get_key();
}
diff --git a/driver/gator_events_armv7.h b/driver/gator_events_armv7.h
new file mode 100644
index 0000000..d5e8d6e
--- /dev/null
+++ b/driver/gator_events_armv7.h
@@ -0,0 +1,38 @@
+/**
+ * Copyright (C) ARM Limited 2011. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef GATOR_EVENTS_ARMV7_H_
+#define GATOR_EVENTS_ARMV7_H_
+
+// Per-CPU PMNC: config reg
+#define PMNC_E (1 << 0) /* Enable all counters */
+#define PMNC_P (1 << 1) /* Reset all counters */
+#define PMNC_C (1 << 2) /* Cycle counter reset */
+#define PMNC_MASK 0x3f /* Mask for writable bits */
+
+// ccnt reg
+#define CCNT_REG (1 << 31)
+
+#define CCNT 0
+#define CNT0 1
+#define CNTMAX (6+1)
+
+// Function prototypes
+extern void armv7_pmnc_write(u32 val);
+extern u32 armv7_pmnc_read(void);
+extern u32 armv7_ccnt_read(u32 reset_value);
+extern u32 armv7_cntn_read(unsigned int cnt, u32 reset_value);
+extern u32 armv7_pmnc_reset_interrupt(void);
+
+// Externed variables
+extern unsigned long pmnc_enabled[CNTMAX];
+extern unsigned long pmnc_event[CNTMAX];
+extern unsigned long pmnc_count[CNTMAX];
+extern unsigned long pmnc_key[CNTMAX];
+
+#endif // GATOR_EVENTS_ARMV7_H_
diff --git a/driver/gator_events_l2c-310.c b/driver/gator_events_l2c-310.c
new file mode 100644
index 0000000..96683b3
--- /dev/null
+++ b/driver/gator_events_l2c-310.c
@@ -0,0 +1,182 @@
+/**
+ * l2c310 (L2 Cache Controller) event counters for gator
+ *
+ * Copyright (C) ARM Limited 2010-2011. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/init.h>
+#include <linux/io.h>
+#include <asm/hardware/cache-l2x0.h>
+
+#include "gator.h"
+
+#define L2C310_COUNTERS_NUM 2
+
+static struct {
+ unsigned long enabled;
+ unsigned long event;
+ unsigned long key;
+} l2c310_counters[L2C310_COUNTERS_NUM];
+
+static int l2c310_buffer[L2C310_COUNTERS_NUM * 2];
+
+static void __iomem *l2c310_base;
+
+
+
+static void gator_events_l2c310_reset_counters(void)
+{
+ u32 val = readl(l2c310_base + L2X0_EVENT_CNT_CTRL);
+
+ val |= ((1 << L2C310_COUNTERS_NUM) - 1) << 1;
+
+ writel(val, l2c310_base + L2X0_EVENT_CNT_CTRL);
+}
+
+
+static int gator_events_l2c310_create_files(struct super_block *sb,
+ struct dentry *root)
+{
+ int i;
+
+ for (i = 0; i < L2C310_COUNTERS_NUM; i++) {
+ char buf[16];
+ struct dentry *dir;
+
+ snprintf(buf, sizeof(buf), "L2C-310_cnt%d", i);
+ dir = gatorfs_mkdir(sb, root, buf);
+ if (WARN_ON(!dir))
+ return -1;
+ gatorfs_create_ulong(sb, dir, "enabled",
+ &l2c310_counters[i].enabled);
+ gatorfs_create_ulong(sb, dir, "event",
+ &l2c310_counters[i].event);
+ gatorfs_create_ro_ulong(sb, dir, "key",
+ &l2c310_counters[i].key);
+ }
+
+ return 0;
+}
+
+static int gator_events_l2c310_start(void)
+{
+ static const unsigned long l2x0_event_cntx_cfg[L2C310_COUNTERS_NUM] = {
+ L2X0_EVENT_CNT0_CFG,
+ L2X0_EVENT_CNT1_CFG,
+ };
+ int i;
+
+ /* Counter event sources */
+ for (i = 0; i < L2C310_COUNTERS_NUM; i++)
+ writel((l2c310_counters[i].event & 0xf) << 2,
+ l2c310_base + l2x0_event_cntx_cfg[i]);
+
+ gator_events_l2c310_reset_counters();
+
+ /* Event counter enable */
+ writel(1, l2c310_base + L2X0_EVENT_CNT_CTRL);
+
+ return 0;
+}
+
+static void gator_events_l2c310_stop(void)
+{
+ /* Event counter disable */
+ writel(0, l2c310_base + L2X0_EVENT_CNT_CTRL);
+}
+
+static int gator_events_l2c310_read(int **buffer)
+{
+ static const unsigned long l2x0_event_cntx_val[L2C310_COUNTERS_NUM] = {
+ L2X0_EVENT_CNT0_VAL,
+ L2X0_EVENT_CNT1_VAL,
+ };
+ int i;
+ int len = 0;
+
+ if (smp_processor_id())
+ return 0;
+
+ for (i = 0; i < L2C310_COUNTERS_NUM; i++) {
+ if (l2c310_counters[i].enabled) {
+ l2c310_buffer[len++] = l2c310_counters[i].key;
+ l2c310_buffer[len++] = readl(l2c310_base +
+ l2x0_event_cntx_val[i]);
+ }
+ }
+
+ /* l2c310 counters are saturating, not wrapping in case of overflow */
+ gator_events_l2c310_reset_counters();
+
+ if (buffer)
+ *buffer = l2c310_buffer;
+
+ return len;
+}
+
+static struct gator_interface gator_events_l2c310_interface = {
+ .create_files = gator_events_l2c310_create_files,
+ .start = gator_events_l2c310_start,
+ .stop = gator_events_l2c310_stop,
+ .read = gator_events_l2c310_read,
+};
+
+static void __maybe_unused gator_events_l2c310_probe(unsigned long phys)
+{
+ if (l2c310_base)
+ return;
+
+ l2c310_base = ioremap(phys, SZ_4K);
+ if (l2c310_base) {
+ u32 cache_id = readl(l2c310_base + L2X0_CACHE_ID);
+
+ if ((cache_id & 0xff0003c0) != 0x410000c0) {
+ iounmap(l2c310_base);
+ l2c310_base = NULL;
+ }
+ }
+}
+
+int gator_events_l2c310_init(void)
+{
+ int i;
+
+ if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9)
+ return -1;
+
+#if defined(CONFIG_ARCH_EXYNOS4)
+ gator_events_l2c310_probe(0xfe600000);
+#endif
+#if defined(CONFIG_ARCH_S5PV310)
+ gator_events_l2c310_probe(0x10502000);
+#endif
+#if defined(CONFIG_ARCH_OMAP4)
+ gator_events_l2c310_probe(0x48242000);
+#endif
+#if defined(CONFIG_ARCH_TEGRA)
+ gator_events_l2c310_probe(0x50043000);
+#endif
+#if defined(CONFIG_ARCH_U8500)
+ gator_events_l2c310_probe(0xa0412000);
+#endif
+#if defined(CONFIG_ARCH_VEXPRESS)
+ // A9x4 core tile (HBI-0191)
+ gator_events_l2c310_probe(0x1e00a000);
+ // New memory map tiles
+ gator_events_l2c310_probe(0x2c0f0000);
+#endif
+ if (!l2c310_base)
+ return -1;
+
+ for (i = 0; i < L2C310_COUNTERS_NUM; i++) {
+ l2c310_counters[i].enabled = 0;
+ l2c310_counters[i].key = gator_events_get_key();
+ }
+
+ return gator_events_install(&gator_events_l2c310_interface);
+}
+gator_events_init(gator_events_l2c310_init);
diff --git a/driver/gator_events_meminfo.c b/driver/gator_events_meminfo.c
index f1595bd..a1a2031 100644
--- a/driver/gator_events_meminfo.c
+++ b/driver/gator_events_meminfo.c
@@ -19,7 +19,7 @@
static ulong meminfo_global_enabled;
static ulong meminfo_enabled[MEMINFO_TOTAL];
static ulong meminfo_key[MEMINFO_TOTAL];
-static int meminfo_buffer[MEMINFO_TOTAL * 2];
+static unsigned long long meminfo_buffer[MEMINFO_TOTAL * 2];
static int meminfo_length = 0;
static unsigned int mem_event = 0;
static bool new_data_avail;
@@ -120,7 +120,8 @@ static void gator_events_meminfo_stop(void)
static void wq_sched_handler(struct work_struct *wsptr)
{
struct sysinfo info;
- int i, len, value;
+ int i, len;
+ unsigned long long value;
meminfo_length = len = 0;
@@ -141,7 +142,7 @@ static void wq_sched_handler(struct work_struct *wsptr)
value = 0;
break;
}
- meminfo_buffer[len++] = meminfo_key[i];
+ meminfo_buffer[len++] = (unsigned long long)meminfo_key[i];
meminfo_buffer[len++] = value;
}
}
@@ -150,7 +151,7 @@ static void wq_sched_handler(struct work_struct *wsptr)
new_data_avail = true;
}
-static int gator_events_meminfo_read(int **buffer)
+static int gator_events_meminfo_read(long long **buffer)
{
static unsigned int last_mem_event = 0;
@@ -177,7 +178,7 @@ static struct gator_interface gator_events_meminfo_interface = {
.create_files = gator_events_meminfo_create_files,
.start = gator_events_meminfo_start,
.stop = gator_events_meminfo_stop,
- .read = gator_events_meminfo_read,
+ .read64 = gator_events_meminfo_read,
};
int gator_events_meminfo_init(void)
diff --git a/driver/gator_events_pl310.c b/driver/gator_events_pl310.c
deleted file mode 100644
index 0ef0cf3..0000000
--- a/driver/gator_events_pl310.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * PL310 (L2 Cache Controller) event counters for gator
- *
- * Copyright (C) ARM Limited 2010-2011. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/init.h>
-#include <linux/io.h>
-#include <asm/hardware/cache-l2x0.h>
-
-#include "gator.h"
-
-#define PL310_COUNTERS_NUM 2
-
-static struct {
- unsigned long enabled;
- unsigned long event;
- unsigned long key;
-} pl310_counters[PL310_COUNTERS_NUM];
-
-static int pl310_buffer[PL310_COUNTERS_NUM * 2];
-
-static void __iomem *pl310_base;
-
-
-
-static void gator_events_pl310_reset_counters(void)
-{
- u32 val = readl(pl310_base + L2X0_EVENT_CNT_CTRL);
-
- val |= ((1 << PL310_COUNTERS_NUM) - 1) << 1;
-
- writel(val, pl310_base + L2X0_EVENT_CNT_CTRL);
-}
-
-
-static int gator_events_pl310_create_files(struct super_block *sb,
- struct dentry *root)
-{
- int i;
-
- for (i = 0; i < PL310_COUNTERS_NUM; i++) {
- char buf[16];
- struct dentry *dir;
-
- snprintf(buf, sizeof(buf), "PL310_cnt%d", i);
- dir = gatorfs_mkdir(sb, root, buf);
- if (WARN_ON(!dir))
- return -1;
- gatorfs_create_ulong(sb, dir, "enabled",
- &pl310_counters[i].enabled);
- gatorfs_create_ulong(sb, dir, "event",
- &pl310_counters[i].event);
- gatorfs_create_ro_ulong(sb, dir, "key",
- &pl310_counters[i].key);
- }
-
- return 0;
-}
-
-static int gator_events_pl310_start(void)
-{
- static const unsigned long l2x0_event_cntx_cfg[PL310_COUNTERS_NUM] = {
- L2X0_EVENT_CNT0_CFG,
- L2X0_EVENT_CNT1_CFG,
- };
- int i;
-
- /* Counter event sources */
- for (i = 0; i < PL310_COUNTERS_NUM; i++)
- writel((pl310_counters[i].event & 0xf) << 2,
- pl310_base + l2x0_event_cntx_cfg[i]);
-
- gator_events_pl310_reset_counters();
-
- /* Event counter enable */
- writel(1, pl310_base + L2X0_EVENT_CNT_CTRL);
-
- return 0;
-}
-
-static void gator_events_pl310_stop(void)
-{
- /* Event counter disable */
- writel(0, pl310_base + L2X0_EVENT_CNT_CTRL);
-}
-
-static int gator_events_pl310_read(int **buffer)
-{
- static const unsigned long l2x0_event_cntx_val[PL310_COUNTERS_NUM] = {
- L2X0_EVENT_CNT0_VAL,
- L2X0_EVENT_CNT1_VAL,
- };
- int i;
- int len = 0;
-
- if (smp_processor_id())
- return 0;
-
- for (i = 0; i < PL310_COUNTERS_NUM; i++) {
- if (pl310_counters[i].enabled) {
- pl310_buffer[len++] = pl310_counters[i].key;
- pl310_buffer[len++] = readl(pl310_base +
- l2x0_event_cntx_val[i]);
- }
- }
-
- /* PL310 counters are saturating, not wrapping in case of overflow */
- gator_events_pl310_reset_counters();
-
- if (buffer)
- *buffer = pl310_buffer;
-
- return len;
-}
-
-static struct gator_interface gator_events_pl310_interface = {
- .create_files = gator_events_pl310_create_files,
- .start = gator_events_pl310_start,
- .stop = gator_events_pl310_stop,
- .read = gator_events_pl310_read,
-};
-
-static void __maybe_unused gator_events_pl310_probe(unsigned long phys)
-{
- if (pl310_base)
- return;
-
- pl310_base = ioremap(phys, SZ_4K);
- if (pl310_base) {
- u32 cache_id = readl(pl310_base + L2X0_CACHE_ID);
-
- if ((cache_id & 0xff0003c0) != 0x410000c0) {
- iounmap(pl310_base);
- pl310_base = NULL;
- }
- }
-}
-
-int gator_events_pl310_init(void)
-{
- int i;
-
-#if defined(CONFIG_ARCH_EXYNOS4)
- gator_events_pl310_probe(0xfe600000);
-#endif
-#if defined(CONFIG_ARCH_OMAP4)
- gator_events_pl310_probe(0x48242000);
-#endif
-#if defined(CONFIG_ARCH_TEGRA)
- gator_events_pl310_probe(0x50043000);
-#endif
-#if defined(CONFIG_ARCH_U8500)
- gator_events_pl310_probe(0xa0412000);
-#endif
-#if defined(CONFIG_ARCH_VEXPRESS) && !defined(CONFIG_ARCH_VEXPRESS_CA15X4)
- // A9x4 core tile (HBI-0191)
- gator_events_pl310_probe(0x1e00a000);
- // New memory map tiles
- gator_events_pl310_probe(0x2c0f0000);
-#endif
- if (!pl310_base)
- return -1;
-
- for (i = 0; i < PL310_COUNTERS_NUM; i++) {
- pl310_counters[i].enabled = 0;
- pl310_counters[i].key = gator_events_get_key();
- }
-
- return gator_events_install(&gator_events_pl310_interface);
-}
-gator_events_init(gator_events_pl310_init);
diff --git a/driver/gator_events_scorpion.c b/driver/gator_events_scorpion.c
index d831a50..f51e292 100644
--- a/driver/gator_events_scorpion.c
+++ b/driver/gator_events_scorpion.c
@@ -12,7 +12,7 @@
#define SCORPIONMP 0x2d
static const char *pmnc_name;
-static int pmnc_count;
+static int pmnc_counters;
// Per-CPU PMNC: config reg
#define PMNC_E (1 << 0) /* Enable all counters */
@@ -32,6 +32,7 @@ static int pmnc_count;
static unsigned long pmnc_enabled[CNTMAX];
static unsigned long pmnc_event[CNTMAX];
+static unsigned long pmnc_count[CNTMAX];
static unsigned long pmnc_key[CNTMAX];
static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
@@ -209,6 +210,10 @@ static const struct scorp_evt sc_evt[] = {
{SCORPION_EXCEPTIONS_UNDERFLOW, 0x80000c00, 4, 0x5d},
{SCORPION_EXCEPTIONS_DENORM, 0x8c000000, 4, 0x5f},
+#ifdef CONFIG_ARCH_MSM_SCORPIONMP
+ {SCORPIONMP_NUM_BARRIERS, 0x80000e00, 3, 0x59},
+ {SCORPIONMP_BARRIER_CYCLES, 0x800e0000, 3, 0x5a},
+#else
{SCORPION_BANK_AB_HIT, 0x80000001, 3, 0x58},
{SCORPION_BANK_AB_ACCESS, 0x80000100, 3, 0x59},
{SCORPION_BANK_CD_HIT, 0x80010000, 3, 0x5a},
@@ -228,6 +233,7 @@ static const struct scorp_evt sc_evt[] = {
{SCORPION_BANK_AB_L2_CASTOUT, 0x80000c00, 3, 0x59},
{SCORPION_BANK_CD_NON_CASTOUT, 0x800c0000, 3, 0x5a},
{SCORPION_BANK_CD_L2_CASTOUT, 0x8c000000, 3, 0x5b},
+#endif
};
static inline void scorpion_pmnc_write(u32 val)
@@ -498,7 +504,7 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den
struct dentry *dir;
int i;
- for (i = 0; i < pmnc_count; i++) {
+ for (i = 0; i < pmnc_counters; i++) {
char buf[40];
if (i == 0) {
snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name);
@@ -510,10 +516,11 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den
return -1;
}
gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]);
+ gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]);
+ gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
if (i > 0) {
gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]);
}
- gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
}
return 0;
@@ -565,6 +572,20 @@ static void gator_events_scorpion_offline(void)
// investigate: need to do the clearpmu() here on each counter?
}
+static int gator_events_scorpion_start(void)
+{
+ int cnt;
+
+ for (cnt = CCNT; cnt < CNTMAX; cnt++) {
+ if (pmnc_count[cnt] > 0) {
+ pr_err("gator: event based sampling not supported on Scorpion cores\n");
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
static void gator_events_scorpion_stop(void)
{
unsigned int cnt;
@@ -572,6 +593,7 @@ static void gator_events_scorpion_stop(void)
for (cnt = CCNT; cnt < CNTMAX; cnt++) {
pmnc_enabled[cnt] = 0;
pmnc_event[cnt] = 0;
+ pmnc_count[cnt] = 0;
}
}
@@ -580,10 +602,10 @@ static int gator_events_scorpion_read(int **buffer)
int cnt, len = 0;
int cpu = smp_processor_id();
- if (!pmnc_count)
+ if (!pmnc_counters)
return 0;
- for (cnt = 0; cnt < pmnc_count; cnt++) {
+ for (cnt = 0; cnt < pmnc_counters; cnt++) {
if (pmnc_enabled[cnt]) {
int value;
if (cnt == CCNT) {
@@ -611,6 +633,7 @@ static int gator_events_scorpion_read(int **buffer)
static struct gator_interface gator_events_scorpion_interface = {
.create_files = gator_events_scorpion_create_files,
+ .start = gator_events_scorpion_start,
.stop = gator_events_scorpion_stop,
.online = gator_events_scorpion_online,
.offline = gator_events_scorpion_offline,
@@ -636,21 +659,22 @@ int gator_events_scorpion_init(void)
switch (gator_cpuid()) {
case SCORPION:
pmnc_name = "Scorpion";
- pmnc_count = 4;
+ pmnc_counters = 4;
break;
case SCORPIONMP:
pmnc_name = "ScorpionMP";
- pmnc_count = 4;
+ pmnc_counters = 4;
break;
default:
return -1;
}
- pmnc_count++; // CNT[n] + CCNT
+ pmnc_counters++; // CNT[n] + CCNT
for (cnt = CCNT; cnt < CNTMAX; cnt++) {
pmnc_enabled[cnt] = 0;
pmnc_event[cnt] = 0;
+ pmnc_count[cnt] = 0;
pmnc_key[cnt] = gator_events_get_key();
}
diff --git a/driver/gator_main.c b/driver/gator_main.c
index 340756e..87e7f2a 100644
--- a/driver/gator_main.c
+++ b/driver/gator_main.c
@@ -7,7 +7,7 @@
*
*/
-static unsigned long gator_protocol_version = 5;
+static unsigned long gator_protocol_version = 6;
#include <linux/slab.h>
#include <linux/cpu.h>
@@ -36,11 +36,13 @@ static unsigned long gator_protocol_version = 5;
#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
#endif
+#if defined (__arm__)
#ifdef CONFIG_SMP
#ifndef CONFIG_LOCAL_TIMERS
#error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
#endif
#endif
+#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
#error kernels prior to 2.6.32 are not supported
@@ -49,22 +51,24 @@ static unsigned long gator_protocol_version = 5;
/******************************************************************************
* DEFINES
******************************************************************************/
-#define BUFFER_SIZE_DEFAULT (256*1024)
-#define SYNC_FREQ_DEFAULT 1000
+#define TIMER_BUFFER_SIZE_DEFAULT (256*1024)
+#define EVENT_BUFFER_SIZE_DEFAULT (128*1024)
#define NO_COOKIE 0UL
#define INVALID_COOKIE ~0UL
-#define PROTOCOL_FRAME ~0
-#define PROTOCOL_START_TICK 1
-#define PROTOCOL_END_TICK 3
-#define PROTOCOL_START_BACKTRACE 5
-#define PROTOCOL_END_BACKTRACE 7
-#define PROTOCOL_COOKIE 9
-#define PROTOCOL_SCHEDULER_TRACE 11
-#define PROTOCOL_COUNTERS 13
-#define PROTOCOL_ANNOTATE 15
-#define PROTOCOL_CPU_SYNC 17
+#define FRAME_HRTIMER 1
+#define FRAME_EVENT 2
+#define FRAME_ANNOTATE 3
+
+#define MESSAGE_COOKIE 1
+#define MESSAGE_COUNTERS 3
+#define MESSAGE_START_BACKTRACE 5
+#define MESSAGE_END_BACKTRACE 7
+#define MESSAGE_SCHEDULER_TRACE 9
+#define MESSAGE_PID_NAME 11
+
+#define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU)
#if defined(__arm__)
#define PC_REG regs->ARM_pc
@@ -72,42 +76,50 @@ static unsigned long gator_protocol_version = 5;
#define PC_REG regs->ip
#endif
+enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};
+
/******************************************************************************
- * PER CPU
+ * Globals
******************************************************************************/
static unsigned long gator_cpu_cores;
-static unsigned long gator_buffer_size;
+static unsigned long userspace_buffer_size;
static unsigned long gator_backtrace_depth;
static unsigned long gator_started;
static unsigned long gator_buffer_opened;
static unsigned long gator_timer_count;
static unsigned long gator_streaming;
-static int gator_master_tick;
static DEFINE_MUTEX(start_mutex);
static DEFINE_MUTEX(gator_buffer_mutex);
unsigned long gator_net_traffic;
+bool event_based_sampling;
#define COMMIT_SIZE 128
#define COMMIT_MASK (COMMIT_SIZE-1)
-static DEFINE_SPINLOCK(gator_commit_lock);
-static int *gator_commit;
-static int gator_commit_read;
-static int gator_commit_write;
+static DEFINE_SPINLOCK(timer_commit_lock);
+static int *gator_commit[NUM_GATOR_BUFS];
+static int gator_commit_read[NUM_GATOR_BUFS];
+static int gator_commit_write[NUM_GATOR_BUFS];
static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
-static DEFINE_PER_CPU(int, gator_cpu_sync);
-static DEFINE_PER_CPU(int, gator_cpu_tick);
static DEFINE_PER_CPU(int, gator_first_time);
+#if LINUX_PMU_SUPPORT
+static void event_buffer_check(int cpu);
+static DEFINE_SPINLOCK(event_commit_lock);
+#endif
+
/******************************************************************************
* Prototypes
******************************************************************************/
-static void gator_buffer_write_packed_int(int cpu, unsigned int x);
-static void gator_buffer_write_string(int cpu, char *x);
+static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
+static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
+static void gator_buffer_write_string(int cpu, int buftype, char *x);
static int gator_write_packed_int(char *buffer, unsigned int x);
-static void gator_add_trace(int cpu, unsigned int address);
+static int gator_write_packed_int64(char *buffer, unsigned long long x);
+static void gator_add_trace(int cpu, int buftype, unsigned int address);
+static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
static uint64_t gator_get_time(void);
/******************************************************************************
@@ -118,6 +130,7 @@ static uint64_t gator_get_time(void);
#include "gator_backtrace.c"
#include "gator_annotate.c"
#include "gator_fs.c"
+#include "gator_ebs.c"
/******************************************************************************
* Misc
@@ -134,204 +147,137 @@ u32 gator_cpuid(void)
/******************************************************************************
* Commit interface
******************************************************************************/
-static int buffer_commit_ready(void)
+static int buffer_commit_ready(int buftype)
{
- return (gator_commit_read != gator_commit_write);
+ return gator_commit_read[buftype] != gator_commit_write[buftype];
}
-static void buffer_commit_read(int *cpu, int *readval, int *writeval)
+static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval)
{
- int read = gator_commit_read;
- *cpu = gator_commit[read+0];
- *readval = gator_commit[read+1];
- *writeval = gator_commit[read+2];
- gator_commit_read = (read + 4) & COMMIT_MASK;
+ int read = gator_commit_read[buftype];
+ *cpu = gator_commit[buftype][read+0];
+ *readval = gator_commit[buftype][read+1];
+ *writeval = gator_commit[buftype][read+2];
+ gator_commit_read[buftype] = (read + 4) & COMMIT_MASK;
}
-static void buffer_commit_write(int cpu, int readval, int writeval) {
- int write = gator_commit_write;
- gator_commit[write+0] = cpu;
- gator_commit[write+1] = readval;
- gator_commit[write+2] = writeval;
- gator_commit_write = (write + 4) & COMMIT_MASK;
+static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) {
+ int write = gator_commit_write[buftype];
+ gator_commit[buftype][write+0] = cpu;
+ gator_commit[buftype][write+1] = readval;
+ gator_commit[buftype][write+2] = writeval;
+ gator_commit_write[buftype] = (write + 4) & COMMIT_MASK;
}
/******************************************************************************
* Buffer management
******************************************************************************/
-static uint32_t use_buffer_size;
-static uint32_t use_buffer_mask;
-static DEFINE_PER_CPU(int, use_buffer_seq);
-static DEFINE_PER_CPU(int, use_buffer_read);
-static DEFINE_PER_CPU(int, use_buffer_write);
-static DEFINE_PER_CPU(char *, use_buffer);
-
-static void gator_buffer_write_packed_int(int cpu, unsigned int x)
-{
- uint32_t write = per_cpu(use_buffer_write, cpu);
- uint32_t mask = use_buffer_mask;
- char *buffer = per_cpu(use_buffer, cpu);
- int write0 = (write + 0) & mask;
- int write1 = (write + 1) & mask;
- int write2 = (write + 2) & mask;
- int write3 = (write + 3) & mask;
- int write4 = (write + 4) & mask;
- int write5 = (write + 5) & mask;
-
- if ((x & 0xffffff80) == 0) {
- buffer[write0] = x & 0x7f;
- per_cpu(use_buffer_write, cpu) = write1;
- } else if ((x & 0xffffc000) == 0) {
- buffer[write0] = x | 0x80;
- buffer[write1] = (x>>7) & 0x7f;
- per_cpu(use_buffer_write, cpu) = write2;
- } else if ((x & 0xffe00000) == 0) {
- buffer[write0] = x | 0x80;
- buffer[write1] = (x>>7) | 0x80;
- buffer[write2] = (x>>14) & 0x7f;
- per_cpu(use_buffer_write, cpu) = write3;
- } else if ((x & 0xf0000000) == 0) {
- buffer[write0] = x | 0x80;
- buffer[write1] = (x>>7) | 0x80;
- buffer[write2] = (x>>14) | 0x80;
- buffer[write3] = (x>>21) & 0x7f;
- per_cpu(use_buffer_write, cpu) = write4;
- } else {
- buffer[write0] = x | 0x80;
- buffer[write1] = (x>>7) | 0x80;
- buffer[write2] = (x>>14) | 0x80;
- buffer[write3] = (x>>21) | 0x80;
- buffer[write4] = (x>>28) & 0x0f;
- per_cpu(use_buffer_write, cpu) = write5;
- }
-}
+static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
+static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
+static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
+static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
+static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
+#include "gator_pack.c"
-static int gator_write_packed_int(char *buffer, unsigned int x)
+static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
{
- if ((x & 0xffffff80) == 0) {
- buffer[0] = x & 0x7f;
- return 1;
- } else if ((x & 0xffffc000) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) & 0x7f;
- return 2;
- } else if ((x & 0xffe00000) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) & 0x7f;
- return 3;
- } else if ((x & 0xf0000000) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) | 0x80;
- buffer[3] = (x>>21) & 0x7f;
- return 4;
- } else {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) | 0x80;
- buffer[3] = (x>>21) | 0x80;
- buffer[4] = (x>>28) & 0x0f;
- return 5;
- }
-}
-
-static void gator_buffer_write_bytes(int cpu, char *x, int len)
-{
- uint32_t write = per_cpu(use_buffer_write, cpu);
- uint32_t mask = use_buffer_mask;
- char *buffer = per_cpu(use_buffer, cpu);
int i;
+ u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
+ u32 mask = gator_buffer_mask[buftype];
+ char* buffer = per_cpu(gator_buffer, cpu)[buftype];
for (i = 0; i < len; i++) {
buffer[write] = x[i];
write = (write + 1) & mask;
}
- per_cpu(use_buffer_write, cpu) = write;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write;
}
-static void gator_buffer_write_string(int cpu, char *x)
+static void gator_buffer_write_string(int cpu, int buftype, char *x)
{
int len = strlen(x);
- gator_buffer_write_packed_int(cpu, len);
- gator_buffer_write_bytes(cpu, x, len);
+ gator_buffer_write_packed_int(cpu, buftype, len);
+ gator_buffer_write_bytes(cpu, buftype, x, len);
}
-static void gator_buffer_header(int cpu)
+static void gator_buffer_header(int cpu, int buftype)
{
- gator_buffer_write_packed_int(cpu, PROTOCOL_FRAME);
- gator_buffer_write_packed_int(cpu, cpu);
- gator_buffer_write_packed_int(cpu, per_cpu(use_buffer_seq, cpu));
- per_cpu(use_buffer_seq, cpu)++;
+ int frame;
+
+ if (buftype == TIMER_BUF)
+ frame = FRAME_HRTIMER;
+ else if (buftype == EVENT_BUF)
+ frame = FRAME_EVENT;
+ else
+ frame = -1;
+
+ gator_buffer_write_packed_int(cpu, buftype, frame);
+ gator_buffer_write_packed_int(cpu, buftype, cpu);
}
-static void gator_buffer_commit(int cpu)
+static void gator_buffer_commit(int cpu, int buftype)
{
- buffer_commit_write(cpu, per_cpu(use_buffer_read, cpu), per_cpu(use_buffer_write, cpu));
- per_cpu(use_buffer_read, cpu) = per_cpu(use_buffer_write, cpu);
- gator_buffer_header(cpu);
+ buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]);
+ per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
+ gator_buffer_header(cpu, buftype);
wake_up(&gator_buffer_wait);
}
-static void gator_buffer_check(int cpu, int tick)
+static void timer_buffer_check(int cpu)
{
- if (!(tick % gator_timer_count)) {
- int c, sync;
- spin_lock(&gator_commit_lock);
- // synchronize, if all online cpus have the same tick waypoint
- sync = per_cpu(gator_cpu_sync, cpu) = per_cpu(gator_cpu_tick, cpu);
- for_each_online_cpu(c) {
- if (sync != per_cpu(gator_cpu_sync, c)) {
- sync = 0;
- break;
- }
- }
- if (sync) {
- gator_buffer_write_packed_int(cpu, PROTOCOL_CPU_SYNC);
- }
- gator_buffer_commit(cpu);
- spin_unlock(&gator_commit_lock);
- } else {
- int available = per_cpu(use_buffer_write, cpu) - per_cpu(use_buffer_read, cpu);
- if (available < 0) {
- available += use_buffer_size;
- }
- if (available >= ((use_buffer_size * 3) / 4)) {
- spin_lock(&gator_commit_lock);
- gator_buffer_commit(cpu);
- spin_unlock(&gator_commit_lock);
- }
+ int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF];
+ if (available < 0) {
+ available += gator_buffer_size[TIMER_BUF];
+ }
+ if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) {
+ spin_lock(&timer_commit_lock);
+ gator_buffer_commit(cpu, TIMER_BUF);
+ spin_unlock(&timer_commit_lock);
+ }
+}
+
+#if LINUX_PMU_SUPPORT
+static void event_buffer_check(int cpu)
+{
+ int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF];
+ if (available < 0) {
+ available += gator_buffer_size[EVENT_BUF];
+ }
+ if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) {
+ spin_lock(&event_commit_lock);
+ gator_buffer_commit(cpu, EVENT_BUF);
+ spin_unlock(&event_commit_lock);
}
}
+#endif
-static void gator_add_trace(int cpu, unsigned int address)
+static void gator_add_trace(int cpu, int buftype, unsigned int address)
{
off_t offset = 0;
- unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
+ unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
offset = address;
}
- gator_buffer_write_packed_int(cpu, offset & ~1);
- gator_buffer_write_packed_int(cpu, cookie);
+ gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
+ gator_buffer_write_packed_int(cpu, buftype, cookie);
}
-static void gator_add_sample(int cpu, struct pt_regs * const regs)
+static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
{
struct module *mod;
unsigned int addr, cookie = 0;
int inKernel = regs ? !user_mode(regs) : 1;
- unsigned long exec_cookie = !inKernel ? get_exec_cookie(cpu, current) : NO_COOKIE;
+ unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
- gator_buffer_write_packed_int(cpu, PROTOCOL_START_BACKTRACE);
-
- // TGID::PID::inKernel
- gator_buffer_write_packed_int(cpu, exec_cookie);
- gator_buffer_write_packed_int(cpu, (unsigned int)current->tgid);
- gator_buffer_write_packed_int(cpu, (unsigned int)current->pid);
- gator_buffer_write_packed_int(cpu, inKernel);
+ gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
+ gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
+ gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
+ gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid);
+ gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
+ gator_buffer_write_packed_int(cpu, buftype, inKernel);
// get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ)
if (regs) {
@@ -339,36 +285,26 @@ static void gator_add_sample(int cpu, struct pt_regs * const regs)
addr = PC_REG;
mod = __module_address(addr);
if (mod) {
- cookie = get_cookie(cpu, current, NULL, mod);
+ cookie = get_cookie(cpu, buftype, current, NULL, mod, true);
addr = addr - (unsigned long)mod->module_core;
}
- gator_buffer_write_packed_int(cpu, addr & ~1);
- gator_buffer_write_packed_int(cpu, cookie);
+ gator_buffer_write_packed_int(cpu, buftype, addr & ~1);
+ gator_buffer_write_packed_int(cpu, buftype, cookie);
} else {
// Cookie+PC
- gator_add_trace(cpu, PC_REG);
+ gator_add_trace(cpu, buftype, PC_REG);
// Backtrace
if (gator_backtrace_depth)
- arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
+ arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
}
}
- gator_buffer_write_packed_int(cpu, PROTOCOL_END_BACKTRACE);
-}
-
-static void gator_write_packet(int cpu, int type, int len, int *buffer)
-{
- int i;
- gator_buffer_write_packed_int(cpu, type);
- gator_buffer_write_packed_int(cpu, len);
- for (i = 0; i < len; i++) {
- gator_buffer_write_packed_int(cpu, buffer[i]);
- }
+ gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
}
/******************************************************************************
- * Interrupt Processing
+ * hrtimer interrupt processing
******************************************************************************/
static LIST_HEAD(gator_events);
@@ -376,7 +312,8 @@ static void gator_timer_interrupt(void)
{
struct pt_regs * const regs = get_irq_regs();
int cpu = smp_processor_id();
- int *buffer, len, tick;
+ int *buffer, len, i, buftype = TIMER_BUF;
+ long long *buffer64;
struct gator_interface *gi;
// check full backtrace has enough space, otherwise may
@@ -391,44 +328,48 @@ static void gator_timer_interrupt(void)
return;
}
- // Header
- gator_buffer_write_packed_int(cpu, PROTOCOL_START_TICK); // Escape
-
// Output scheduler
- len = gator_trace_sched_read(&buffer);
+ len = gator_trace_sched_read(&buffer64);
if (len > 0) {
- gator_write_packet(cpu, PROTOCOL_SCHEDULER_TRACE, len, buffer);
+ gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
+ gator_buffer_write_packed_int(cpu, buftype, len);
+ for (i = 0; i < len; i++) {
+ gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
+ }
}
// Output counters
+ gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
+ gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
list_for_each_entry(gi, &gator_events, list) {
if (gi->read) {
len = gi->read(&buffer);
+ if (len > 0) {
+ gator_buffer_write_packed_int(cpu, buftype, len);
+ for (i = 0; i < len; i++) {
+ gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
+ }
+ }
+ } else if (gi->read64) {
+ len = gi->read64(&buffer64);
if (len > 0)
- gator_write_packet(cpu, PROTOCOL_COUNTERS, len, buffer);
+ gator_buffer_write_packed_int(cpu, buftype, len);
+ for (i = 0; i < len; i++) {
+ gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
+ }
}
}
+ gator_buffer_write_packed_int(cpu, buftype, 0);
// Output backtrace
- gator_add_sample(cpu, regs);
-
- // Timer Tick
- tick = per_cpu(gator_cpu_tick, cpu);
- if (tick == gator_master_tick) {
- tick++;
- per_cpu(gator_cpu_tick, cpu) = gator_master_tick = tick;
- } else {
- per_cpu(gator_cpu_tick, cpu) = tick = gator_master_tick;
+ if (!event_based_sampling) {
+ gator_add_sample(cpu, buftype, regs);
}
- gator_write_packet(cpu, PROTOCOL_END_TICK, 1, &tick);
// Check and commit; generally, commit is set to occur once per second
- gator_buffer_check(cpu, tick);
+ timer_buffer_check(cpu);
}
-/******************************************************************************
- * hrtimer
- ******************************************************************************/
DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
DEFINE_PER_CPU(int, hrtimer_is_active);
static int hrtimer_running;
@@ -454,7 +395,9 @@ static void __gator_timer_offline(void *unused)
struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
hrtimer_cancel(hrtimer);
per_cpu(hrtimer_is_active, cpu) = 0;
- gator_buffer_commit(cpu);
+ gator_buffer_commit(cpu, TIMER_BUF);
+ if (event_based_sampling)
+ gator_buffer_commit(cpu, EVENT_BUF);
// offline any events
list_for_each_entry(gi, &gator_events, list)
@@ -469,10 +412,6 @@ static void gator_timer_offline(void)
hrtimer_running = 0;
on_each_cpu(__gator_timer_offline, NULL, 1);
-
- // output a final sync point
- gator_buffer_write_packed_int(0, PROTOCOL_CPU_SYNC);
- gator_buffer_commit(0);
}
}
@@ -485,7 +424,6 @@ static void __gator_timer_online(void *unused)
hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
hrtimer->function = gator_hrtimer_notify;
hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
- per_cpu(gator_cpu_tick, cpu) = 0;
per_cpu(gator_first_time, cpu) = 1;
per_cpu(hrtimer_is_active, cpu) = 1;
@@ -499,10 +437,10 @@ static void __gator_timer_online(void *unused)
int gator_timer_online(unsigned long setup)
{
if (!setup) {
- pr_err("gator: cannot start due to a system tick value of zero");
+ pr_err("gator: cannot start due to a system tick value of zero\n");
return -1;
} else if (hrtimer_running) {
- pr_notice("gator: high res timer already running");
+ pr_notice("gator: high res timer already running\n");
return 0;
}
@@ -512,7 +450,6 @@ int gator_timer_online(unsigned long setup)
profiling_interval = ns_to_ktime(1000000000UL / setup);
// timer interrupt
- gator_master_tick = 0;
on_each_cpu(__gator_timer_online, NULL, 1);
return 0;
@@ -623,10 +560,14 @@ static int gator_start(void)
}
}
+ if (gator_event_sampling_start())
+ goto event_sampling_failure;
if (gator_annotate_start())
goto annotate_failure;
if (gator_trace_sched_start())
goto sched_failure;
+ if (cookies_initialize())
+ goto cookies_failure;
if (gator_timer_online(gator_timer_count))
goto timer_failure;
if (gator_notifier_start())
@@ -637,10 +578,18 @@ static int gator_start(void)
notifier_failure:
gator_timer_offline();
timer_failure:
+ cookies_release();
+cookies_failure:
gator_trace_sched_stop();
sched_failure:
gator_annotate_stop();
annotate_failure:
+ gator_event_sampling_stop();
+event_sampling_failure:
+ // stop all events
+ list_for_each_entry(gi, &gator_events, list)
+ if (gi->stop)
+ gi->stop();
events_failure:
return -1;
@@ -655,12 +604,13 @@ static void gator_stop(void)
if (gi->stop)
gi->stop();
+ gator_event_sampling_stop();
gator_annotate_stop();
gator_trace_sched_stop();
// stop all interrupt callback reads before tearing down other interfaces
+ gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined
gator_timer_offline();
- gator_notifier_stop();
}
static void gator_exit(void)
@@ -675,42 +625,45 @@ static void gator_exit(void)
static int gator_op_setup(void)
{
int err = 0;
- int cpu;
+ int cpu, i;
mutex_lock(&start_mutex);
- use_buffer_size = gator_buffer_size;
- use_buffer_mask = use_buffer_size - 1;
+ gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
+ gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;
// must be a power of 2
- if (use_buffer_size & (use_buffer_size - 1)) {
+ if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
err = -ENOEXEC;
goto setup_error;
}
- gator_net_traffic = 0;
+ gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
+ gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
- gator_commit_read = gator_commit_write = 0;
- gator_commit = vmalloc(COMMIT_SIZE * sizeof(int));
- if (!gator_commit) {
- err = -ENOMEM;
- goto setup_error;
- }
+ gator_net_traffic = 0;
- for_each_present_cpu(cpu) {
- per_cpu(use_buffer, cpu) = vmalloc(use_buffer_size);
- if (!per_cpu(use_buffer, cpu)) {
+ // Initialize per buffer variables
+ for (i = 0; i < NUM_GATOR_BUFS; i++) {
+ gator_commit_read[i] = gator_commit_write[i] = 0;
+ gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int));
+ if (!gator_commit[i]) {
err = -ENOMEM;
goto setup_error;
}
- per_cpu(gator_cpu_sync, cpu) = 0;
- per_cpu(gator_cpu_tick, cpu) = 0;
+ // Initialize percpu per buffer variables
+ for_each_present_cpu(cpu) {
+ per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
+ if (!per_cpu(gator_buffer, cpu)[i]) {
+ err = -ENOMEM;
+ goto setup_error;
+ }
- per_cpu(use_buffer_seq, cpu) = 0;
- per_cpu(use_buffer_read, cpu) = 0;
- per_cpu(use_buffer_write, cpu) = 0;
- gator_buffer_header(cpu);
+ per_cpu(gator_buffer_read, cpu)[i] = 0;
+ per_cpu(gator_buffer_write, cpu)[i] = 0;
+ gator_buffer_header(cpu, i);
+ }
}
setup_error:
@@ -725,7 +678,7 @@ static int gator_op_start(void)
mutex_lock(&start_mutex);
- if (gator_started || gator_start() || cookies_initialize())
+ if (gator_started || gator_start())
err = -EINVAL;
else
gator_started = 1;
@@ -757,20 +710,25 @@ static void gator_op_stop(void)
static void gator_shutdown(void)
{
- int cpu;
+ int cpu, i;
mutex_lock(&start_mutex);
- vfree(gator_commit);
- gator_commit = NULL;
+ gator_annotate_shutdown();
+
+ for (i = 0; i < NUM_GATOR_BUFS; i++) {
+ vfree(gator_commit[i]);
+ gator_commit[i] = NULL;
+ }
for_each_present_cpu(cpu) {
mutex_lock(&gator_buffer_mutex);
- vfree(per_cpu(use_buffer, cpu));
- per_cpu(use_buffer, cpu) = NULL;
- per_cpu(use_buffer_seq, cpu) = 0;
- per_cpu(use_buffer_read, cpu) = 0;
- per_cpu(use_buffer_write, cpu) = 0;
+ for (i = 0; i < NUM_GATOR_BUFS; i++) {
+ vfree(per_cpu(gator_buffer, cpu)[i]);
+ per_cpu(gator_buffer, cpu)[i] = NULL;
+ per_cpu(gator_buffer_read, cpu)[i] = 0;
+ per_cpu(gator_buffer_write, cpu)[i] = 0;
+ }
mutex_unlock(&gator_buffer_mutex);
}
@@ -825,7 +783,7 @@ static const struct file_operations enable_fops = {
.write = enable_write,
};
-static int event_buffer_open(struct inode *inode, struct file *file)
+static int userspace_buffer_open(struct inode *inode, struct file *file)
{
int err = -EPERM;
@@ -849,7 +807,7 @@ fail:
return err;
}
-static int event_buffer_release(struct inode *inode, struct file *file)
+static int userspace_buffer_release(struct inode *inode, struct file *file)
{
gator_op_stop();
gator_shutdown();
@@ -857,55 +815,59 @@ static int event_buffer_release(struct inode *inode, struct file *file)
return 0;
}
-static ssize_t event_buffer_read(struct file *file, char __user *buf,
+static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
size_t count, loff_t *offset)
{
int retval = -EINVAL;
int commit, length1, length2, read;
- char *buffer1, *buffer2;
- char annotate_header[6];
- int cpu;
+ char *buffer1;
+ char *buffer2 = NULL;
+ int cpu, i;
/* do not handle partial reads */
- if (count != use_buffer_size || *offset)
+ if (count != userspace_buffer_size || *offset)
return -EINVAL;
// sleep until the condition is true or a signal is received
// the condition is checked each time gator_buffer_wait is woken up
- wait_event_interruptible(gator_buffer_wait, buffer_commit_ready() || gator_annotate_ready() || !gator_started);
+ wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started);
if (signal_pending(current))
return -EINTR;
+ length2 = 0;
retval = -EFAULT;
mutex_lock(&gator_buffer_mutex);
- if (buffer_commit_ready()) {
- buffer_commit_read(&cpu, &read, &commit);
+ i = -1;
+ if (buffer_commit_ready(TIMER_BUF)) {
+ i = TIMER_BUF;
+ } else if (buffer_commit_ready(EVENT_BUF)) {
+ i = EVENT_BUF;
+ }
+
+ if (i != -1) {
+ buffer_commit_read(&cpu, i, &read, &commit);
/* May happen if the buffer is freed during pending reads. */
- if (!per_cpu(use_buffer, cpu)) {
+ if (!per_cpu(gator_buffer, cpu)[i]) {
retval = -EFAULT;
goto out;
}
/* determine the size of two halves */
length1 = commit - read;
- length2 = 0;
- buffer1 = &(per_cpu(use_buffer, cpu)[read]);
- buffer2 = &(per_cpu(use_buffer, cpu)[0]);
+ buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]);
+ buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]);
if (length1 < 0) {
- length1 = use_buffer_size - read;
+ length1 = gator_buffer_size[i] - read;
length2 = commit;
}
} else if (gator_annotate_ready()) {
- length2 = gator_annotate_read(&buffer2);
- if (!length2)
+ length1 = gator_annotate_read(&buffer1);
+ if (!length1)
goto out;
- annotate_header[0] = PROTOCOL_ANNOTATE;
- length1 = gator_write_packed_int(&annotate_header[1], length2) + 1;
- buffer1 = annotate_header;
} else {
retval = 0;
goto out;
@@ -939,9 +901,9 @@ out:
}
const struct file_operations gator_event_buffer_fops = {
- .open = event_buffer_open,
- .release = event_buffer_release,
- .read = event_buffer_read,
+ .open = userspace_buffer_open,
+ .release = userspace_buffer_release,
+ .read = userspace_buffer_read,
};
static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
@@ -974,17 +936,6 @@ static const struct file_operations depth_fops = {
.write = depth_write
};
-static const char gator_cpu_type[] = "gator";
-
-static ssize_t cpu_type_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
-{
- return gatorfs_str_to_user(gator_cpu_type, buf, count, offset);
-}
-
-static const struct file_operations cpu_type_fops = {
- .read = cpu_type_read,
-};
-
void gator_op_create_files(struct super_block *sb, struct dentry *root)
{
struct dentry *dir;
@@ -996,15 +947,14 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root)
for_each_present_cpu(cpu) {
gator_cpu_cores++;
}
- gator_buffer_size = BUFFER_SIZE_DEFAULT;
+ userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT;
gator_streaming = 1;
gatorfs_create_file(sb, root, "enable", &enable_fops);
gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
- gatorfs_create_file(sb, root, "cpu_type", &cpu_type_fops);
gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
- gatorfs_create_ulong(sb, root, "buffer_size", &gator_buffer_size);
+ gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
@@ -1033,17 +983,11 @@ static int __init gator_module_init(void)
return -1;
}
-#ifdef GATOR_DEBUG
- pr_err("gator_module_init");
-#endif
return 0;
}
static void __exit gator_module_exit(void)
{
-#ifdef GATOR_DEBUG
- pr_err("gator_module_exit");
-#endif
tracepoint_synchronize_unregister();
gatorfs_unregister();
gator_exit();
diff --git a/driver/gator_pack.c b/driver/gator_pack.c
new file mode 100644
index 0000000..fbab220
--- /dev/null
+++ b/driver/gator_pack.c
@@ -0,0 +1,262 @@
+/**
+ * Copyright (C) ARM Limited 2010-2011. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x)
+{
+ uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype];
+ uint32_t mask = gator_buffer_mask[buftype];
+ char *buffer = per_cpu(gator_buffer, cpu)[buftype];
+ int write0 = (write + 0) & mask;
+ int write1 = (write + 1) & mask;
+
+ if ((x & 0xffffff80) == 0) {
+ buffer[write0] = x & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write1;
+ } else if ((x & 0xffffc000) == 0) {
+ int write2 = (write + 2) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write2;
+ } else if ((x & 0xffe00000) == 0) {
+ int write2 = (write + 2) & mask;
+ int write3 = (write + 3) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) | 0x80;
+ buffer[write2] = (x>>14) & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write3;
+ } else if ((x & 0xf0000000) == 0) {
+ int write2 = (write + 2) & mask;
+ int write3 = (write + 3) & mask;
+ int write4 = (write + 4) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) | 0x80;
+ buffer[write2] = (x>>14) | 0x80;
+ buffer[write3] = (x>>21) & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write4;
+ } else {
+ int write2 = (write + 2) & mask;
+ int write3 = (write + 3) & mask;
+ int write4 = (write + 4) & mask;
+ int write5 = (write + 5) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) | 0x80;
+ buffer[write2] = (x>>14) | 0x80;
+ buffer[write3] = (x>>21) | 0x80;
+ buffer[write4] = (x>>28) & 0x0f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write5;
+ }
+}
+
+static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x)
+{
+ uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype];
+ uint32_t mask = gator_buffer_mask[buftype];
+ char *buffer = per_cpu(gator_buffer, cpu)[buftype];
+ int write0 = (write + 0) & mask;
+ int write1 = (write + 1) & mask;
+
+ if ((x & 0xffffffffffffff80LL) == 0) {
+ buffer[write0] = x & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write1;
+ } else if ((x & 0xffffffffffffc000LL) == 0) {
+ int write2 = (write + 2) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write2;
+ } else if ((x & 0xffffffffffe00000LL) == 0) {
+ int write2 = (write + 2) & mask;
+ int write3 = (write + 3) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) | 0x80;
+ buffer[write2] = (x>>14) & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write3;
+ } else if ((x & 0xfffffffff0000000LL) == 0) {
+ int write2 = (write + 2) & mask;
+ int write3 = (write + 3) & mask;
+ int write4 = (write + 4) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) | 0x80;
+ buffer[write2] = (x>>14) | 0x80;
+ buffer[write3] = (x>>21) & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write4;
+ } else if ((x & 0xfffffff800000000LL) == 0) {
+ int write2 = (write + 2) & mask;
+ int write3 = (write + 3) & mask;
+ int write4 = (write + 4) & mask;
+ int write5 = (write + 5) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) | 0x80;
+ buffer[write2] = (x>>14) | 0x80;
+ buffer[write3] = (x>>21) | 0x80;
+ buffer[write4] = (x>>28) & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write5;
+ } else if ((x & 0xfffffc0000000000LL) == 0) {
+ int write2 = (write + 2) & mask;
+ int write3 = (write + 3) & mask;
+ int write4 = (write + 4) & mask;
+ int write5 = (write + 5) & mask;
+ int write6 = (write + 6) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) | 0x80;
+ buffer[write2] = (x>>14) | 0x80;
+ buffer[write3] = (x>>21) | 0x80;
+ buffer[write4] = (x>>28) | 0x80;
+ buffer[write5] = (x>>35) & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write6;
+ } else if ((x & 0xfffe000000000000LL) == 0) {
+ int write2 = (write + 2) & mask;
+ int write3 = (write + 3) & mask;
+ int write4 = (write + 4) & mask;
+ int write5 = (write + 5) & mask;
+ int write6 = (write + 6) & mask;
+ int write7 = (write + 7) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) | 0x80;
+ buffer[write2] = (x>>14) | 0x80;
+ buffer[write3] = (x>>21) | 0x80;
+ buffer[write4] = (x>>28) | 0x80;
+ buffer[write5] = (x>>35) | 0x80;
+ buffer[write6] = (x>>42) & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write7;
+ } else if ((x & 0xff00000000000000LL) == 0) {
+ int write2 = (write + 2) & mask;
+ int write3 = (write + 3) & mask;
+ int write4 = (write + 4) & mask;
+ int write5 = (write + 5) & mask;
+ int write6 = (write + 6) & mask;
+ int write7 = (write + 7) & mask;
+ int write8 = (write + 8) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) | 0x80;
+ buffer[write2] = (x>>14) | 0x80;
+ buffer[write3] = (x>>21) | 0x80;
+ buffer[write4] = (x>>28) | 0x80;
+ buffer[write5] = (x>>35) | 0x80;
+ buffer[write6] = (x>>42) | 0x80;
+ buffer[write7] = (x>>49) & 0x7f;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write8;
+ } else {
+ int write2 = (write + 2) & mask;
+ int write3 = (write + 3) & mask;
+ int write4 = (write + 4) & mask;
+ int write5 = (write + 5) & mask;
+ int write6 = (write + 6) & mask;
+ int write7 = (write + 7) & mask;
+ int write8 = (write + 8) & mask;
+ int write9 = (write + 9) & mask;
+ buffer[write0] = x | 0x80;
+ buffer[write1] = (x>>7) | 0x80;
+ buffer[write2] = (x>>14) | 0x80;
+ buffer[write3] = (x>>21) | 0x80;
+ buffer[write4] = (x>>28) | 0x80;
+ buffer[write5] = (x>>35) | 0x80;
+ buffer[write6] = (x>>42) | 0x80;
+ buffer[write7] = (x>>49) | 0x80;
+ buffer[write8] = (x>>56) & 0xff;
+ per_cpu(gator_buffer_write, cpu)[buftype] = write9;
+ }
+}
+
+static int gator_write_packed_int(char *buffer, unsigned int x)
+{
+ if ((x & 0xffffff80) == 0) {
+ buffer[0] = x & 0x7f;
+ return 1;
+ } else if ((x & 0xffffc000) == 0) {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) & 0x7f;
+ return 2;
+ } else if ((x & 0xffe00000) == 0) {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) | 0x80;
+ buffer[2] = (x>>14) & 0x7f;
+ return 3;
+ } else if ((x & 0xf0000000) == 0) {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) | 0x80;
+ buffer[2] = (x>>14) | 0x80;
+ buffer[3] = (x>>21) & 0x7f;
+ return 4;
+ } else {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) | 0x80;
+ buffer[2] = (x>>14) | 0x80;
+ buffer[3] = (x>>21) | 0x80;
+ buffer[4] = (x>>28) & 0x0f;
+ return 5;
+ }
+}
+
+static int gator_write_packed_int64(char *buffer, unsigned long long x)
+{
+ if ((x & 0xffffffffffffff80LL) == 0) {
+ buffer[0] = x & 0x7f;
+ return 1;
+ } else if ((x & 0xffffffffffffc000LL) == 0) {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) & 0x7f;
+ return 2;
+ } else if ((x & 0xffffffffffe00000LL) == 0) {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) | 0x80;
+ buffer[2] = (x>>14) & 0x7f;
+ return 3;
+ } else if ((x & 0xfffffffff0000000LL) == 0) {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) | 0x80;
+ buffer[2] = (x>>14) | 0x80;
+ buffer[3] = (x>>21) & 0x7f;
+ return 4;
+ } else if ((x & 0xfffffff800000000LL) == 0) {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) | 0x80;
+ buffer[2] = (x>>14) | 0x80;
+ buffer[3] = (x>>21) | 0x80;
+ buffer[4] = (x>>28) & 0x7f;
+ return 5;
+ } else if ((x & 0xfffffc0000000000LL) == 0) {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) | 0x80;
+ buffer[2] = (x>>14) | 0x80;
+ buffer[3] = (x>>21) | 0x80;
+ buffer[4] = (x>>28) | 0x80;
+ buffer[5] = (x>>35) & 0x7f;
+ return 6;
+ } else if ((x & 0xfffe000000000000LL) == 0) {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) | 0x80;
+ buffer[2] = (x>>14) | 0x80;
+ buffer[3] = (x>>21) | 0x80;
+ buffer[4] = (x>>28) | 0x80;
+ buffer[5] = (x>>35) | 0x80;
+ buffer[6] = (x>>42) & 0x7f;
+ return 7;
+ } else if ((x & 0xff00000000000000LL) == 0) {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) | 0x80;
+ buffer[2] = (x>>14) | 0x80;
+ buffer[3] = (x>>21) | 0x80;
+ buffer[4] = (x>>28) | 0x80;
+ buffer[5] = (x>>35) | 0x80;
+ buffer[6] = (x>>42) | 0x80;
+ buffer[7] = (x>>49) & 0x7f;
+ return 8;
+ } else {
+ buffer[0] = x | 0x80;
+ buffer[1] = (x>>7) | 0x80;
+ buffer[2] = (x>>14) | 0x80;
+ buffer[3] = (x>>21) | 0x80;
+ buffer[4] = (x>>28) | 0x80;
+ buffer[5] = (x>>35) | 0x80;
+ buffer[6] = (x>>42) | 0x80;
+ buffer[7] = (x>>49) | 0x80;
+ buffer[8] = (x>>56) & 0xff;
+ return 9;
+ }
+}
diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c
index 19d8d89..7c0bd47 100644
--- a/driver/gator_trace_sched.c
+++ b/driver/gator_trace_sched.c
@@ -10,36 +10,89 @@
#include <trace/events/sched.h>
#include "gator.h"
-#define SCHED_TIMER_EVENT 0
-#define SCHED_WAIT_TASK 1
-#define SCHED_WAKEUP 2
-#define SCHED_WAKEUP_NEW 3
-#define SCHED_SWITCH 4
-#define SCHED_MIGRATE_TASK 5
-#define SCHED_PROCESS_FREE 6
-#define SCHED_PROCESS_EXIT 7
-#define SCHED_PROCESS_WAIT 8
-#define SCHED_PROCESS_FORK 9
#define SCHED_OVERFLOW -1
+#define SCHED_SWITCH 1
+#define SCHED_PROCESS_FREE 2
-#define SCHEDSIZE (16*1024)
+#define FIELD_TYPE 0
+#define FIELD_TIME 1
+#define FIELD_PARAM1 2
+#define FIELD_PARAM2 3
+#define FIELD_PARAM3 4
+#define FIELDS_PER_SCHED 5
-static DEFINE_PER_CPU(int *[2], theSchedBuf);
+#define SCHEDSIZE (8*1024)
+#define TASK_MAP_ENTRIES 1024 /* must be power of 2 */
+#define TASK_MAX_COLLISIONS 2
+
+static DEFINE_PER_CPU(uint64_t *[2], theSchedBuf);
static DEFINE_PER_CPU(int, theSchedSel);
static DEFINE_PER_CPU(int, theSchedPos);
static DEFINE_PER_CPU(int, theSchedErr);
+static DEFINE_PER_CPU(uint64_t *, taskname_keys);
+
+void emit_pid_name(uint64_t time, struct task_struct* task)
+{
+ bool found = false;
+ unsigned long flags;
+ char taskcomm[TASK_COMM_LEN + 3];
+ int x, cpu = smp_processor_id();
+ uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]);
+ uint64_t value;
+
+ value = gator_chksum_crc32(task->comm);
+ value = (value << 32) | (uint32_t)task->pid;
+
+ // determine if the thread name was emitted already
+ for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
+ if (keys[x] == value) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ // shift values, new value always in front
+ uint64_t oldv, newv = value;
+ for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
+ oldv = keys[x];
+ keys[x] = newv;
+ newv = oldv;
+ }
+
+ // emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions
+ if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1)
+ // append ellipses if task->comm has length of TASK_COMM_LEN - 1
+ strcat(taskcomm, "...");
+
+ // disable interrupts to synchronize with hrtimer populating timer buf
+ local_irq_save(flags);
+ gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_PID_NAME);
+ gator_buffer_write_packed_int64(cpu, TIMER_BUF, time);
+ gator_buffer_write_packed_int(cpu, TIMER_BUF, task->pid);
+ gator_buffer_write_string(cpu, TIMER_BUF, taskcomm);
+ local_irq_restore(flags);
+ }
+}
static void probe_sched_write(int type, int param1, int param2, int param3)
{
unsigned long flags;
int cpu = smp_processor_id();
uint64_t time = gator_get_time();
- int *schedBuf;
- int schedPos;
+ uint64_t *schedBuf;
+ int schedPos, cookie = param3;
if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)])
return;
+ if (param3) {
+ // do as much work as possible before disabling interrupts
+ struct task_struct *task = (struct task_struct *)param3;
+ cookie = get_exec_cookie(cpu, TIMER_BUF, task);
+ emit_pid_name(time, task);
+ }
+
// disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used
local_irq_save(flags);
@@ -48,88 +101,37 @@ static void probe_sched_write(int type, int param1, int param2, int param3)
if (schedPos < (SCHEDSIZE-100)) {
// capture
- schedBuf[schedPos+0] = type;
- schedBuf[schedPos+1] = (int)time;
- schedBuf[schedPos+2] = (int)(time >> 32);
- schedBuf[schedPos+3] = param1;
- schedBuf[schedPos+4] = param2;
- schedBuf[schedPos+5] = param3;
- per_cpu(theSchedPos, cpu) = schedPos + 6;
+ schedBuf[schedPos+FIELD_TYPE] = type;
+ schedBuf[schedPos+FIELD_TIME] = time;
+ schedBuf[schedPos+FIELD_PARAM1] = param1;
+ schedBuf[schedPos+FIELD_PARAM2] = param2;
+ schedBuf[schedPos+FIELD_PARAM3] = cookie;
+ per_cpu(theSchedPos, cpu) = schedPos + FIELDS_PER_SCHED;
} else if (!per_cpu(theSchedErr, cpu)) {
per_cpu(theSchedErr, cpu) = 1;
- schedBuf[schedPos+0] = SCHED_OVERFLOW;
- schedBuf[schedPos+1] = 0;
- schedBuf[schedPos+2] = 0;
- schedBuf[schedPos+3] = 0;
- schedBuf[schedPos+4] = 0;
- schedBuf[schedPos+5] = 0;
- per_cpu(theSchedPos, cpu) = schedPos + 6;
+ schedBuf[schedPos+FIELD_TYPE] = SCHED_OVERFLOW;
+ schedBuf[schedPos+FIELD_TIME] = time;
+ schedBuf[schedPos+FIELD_PARAM1] = 0;
+ schedBuf[schedPos+FIELD_PARAM2] = 0;
+ schedBuf[schedPos+FIELD_PARAM3] = 0;
+ per_cpu(theSchedPos, cpu) = schedPos + FIELDS_PER_SCHED;
pr_debug("gator: tracepoint overflow\n");
}
local_irq_restore(flags);
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
-GATOR_DEFINE_PROBE(sched_wait_task, TP_PROTO(struct rq *rq, struct task_struct *p))
-#else
-GATOR_DEFINE_PROBE(sched_wait_task, TP_PROTO(struct task_struct *p))
-#endif
-{
- probe_sched_write(SCHED_WAIT_TASK, 0, p->pid, 0);
-}
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
-GATOR_DEFINE_PROBE(sched_wakeup, TP_PROTO(struct rq *rq, struct task_struct *p, int success))
-#else
-GATOR_DEFINE_PROBE(sched_wakeup, TP_PROTO(struct task_struct *p, int success))
-#endif
-{
- if (success)
- probe_sched_write(SCHED_WAKEUP, 0, p->pid, 0);
-}
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
-GATOR_DEFINE_PROBE(sched_wakeup_new, TP_PROTO(struct rq *rq, struct task_struct *p, int success))
-#else
-GATOR_DEFINE_PROBE(sched_wakeup_new, TP_PROTO(struct task_struct *p, int success))
-#endif
-{
- if (success)
- probe_sched_write(SCHED_WAKEUP_NEW, 0, p->tgid, p->pid);
-}
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next))
#else
GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next))
#endif
{
- probe_sched_write(SCHED_SWITCH, (int)next, next->tgid, next->pid);
-}
-
-GATOR_DEFINE_PROBE(sched_migrate_task, TP_PROTO(struct task_struct *p, int dest_cpu))
-{
- probe_sched_write(SCHED_MIGRATE_TASK, 0, dest_cpu, p->pid);
+ probe_sched_write(SCHED_SWITCH, next->pid, next->tgid, (int)next);
}
GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p))
{
- probe_sched_write(SCHED_PROCESS_FREE, 0, p->pid, 0);
-}
-
-GATOR_DEFINE_PROBE(sched_process_exit, TP_PROTO(struct task_struct *p))
-{
- probe_sched_write(SCHED_PROCESS_EXIT, 0, p->pid, 0);
-}
-
-GATOR_DEFINE_PROBE(sched_process_wait, TP_PROTO(struct pid *pid))
-{
- probe_sched_write(SCHED_PROCESS_WAIT, 0, pid_nr(pid), 0);
-}
-
-GATOR_DEFINE_PROBE(sched_process_fork, TP_PROTO(struct task_struct *parent, struct task_struct *child))
-{
- probe_sched_write(SCHED_PROCESS_FORK, (int)child, parent->pid, child->pid);
+ probe_sched_write(SCHED_PROCESS_FREE, p->pid, 0, 0);
}
int gator_trace_sched_init(void)
@@ -139,59 +141,37 @@ int gator_trace_sched_init(void)
int gator_trace_sched_start(void)
{
- int cpu;
+ int cpu, size;
for_each_present_cpu(cpu) {
per_cpu(theSchedSel, cpu) = 0;
per_cpu(theSchedPos, cpu) = 0;
per_cpu(theSchedErr, cpu) = 0;
- per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(int), GFP_KERNEL);
- per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(int), GFP_KERNEL);
+ per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL);
+ per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL);
if (!per_cpu(theSchedBuf, cpu))
return -1;
+
+ size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t);
+ per_cpu(taskname_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL);
+ if (!per_cpu(taskname_keys, cpu))
+ return -1;
+ memset(per_cpu(taskname_keys, cpu), 0, size);
}
// register tracepoints
- if (GATOR_REGISTER_TRACE(sched_wait_task))
- goto fail_sched_wait_task;
- if (GATOR_REGISTER_TRACE(sched_wakeup))
- goto fail_sched_wakeup;
- if (GATOR_REGISTER_TRACE(sched_wakeup_new))
- goto fail_sched_wakeup_new;
if (GATOR_REGISTER_TRACE(sched_switch))
goto fail_sched_switch;
- if (GATOR_REGISTER_TRACE(sched_migrate_task))
- goto fail_sched_migrate_task;
if (GATOR_REGISTER_TRACE(sched_process_free))
goto fail_sched_process_free;
- if (GATOR_REGISTER_TRACE(sched_process_exit))
- goto fail_sched_process_exit;
- if (GATOR_REGISTER_TRACE(sched_process_wait))
- goto fail_sched_process_wait;
- if (GATOR_REGISTER_TRACE(sched_process_fork))
- goto fail_sched_process_fork;
pr_debug("gator: registered tracepoints\n");
return 0;
// unregister tracepoints on error
-fail_sched_process_fork:
- GATOR_UNREGISTER_TRACE(sched_process_wait);
-fail_sched_process_wait:
- GATOR_UNREGISTER_TRACE(sched_process_exit);
-fail_sched_process_exit:
- GATOR_UNREGISTER_TRACE(sched_process_free);
fail_sched_process_free:
- GATOR_UNREGISTER_TRACE(sched_migrate_task);
-fail_sched_migrate_task:
GATOR_UNREGISTER_TRACE(sched_switch);
fail_sched_switch:
- GATOR_UNREGISTER_TRACE(sched_wakeup_new);
-fail_sched_wakeup_new:
- GATOR_UNREGISTER_TRACE(sched_wakeup);
-fail_sched_wakeup:
- GATOR_UNREGISTER_TRACE(sched_wait_task);
-fail_sched_wait_task:
pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");
return -1;
@@ -200,15 +180,8 @@ fail_sched_wait_task:
void gator_trace_sched_stop(void)
{
int cpu;
- GATOR_UNREGISTER_TRACE(sched_wait_task);
- GATOR_UNREGISTER_TRACE(sched_wakeup);
- GATOR_UNREGISTER_TRACE(sched_wakeup_new);
GATOR_UNREGISTER_TRACE(sched_switch);
- GATOR_UNREGISTER_TRACE(sched_migrate_task);
GATOR_UNREGISTER_TRACE(sched_process_free);
- GATOR_UNREGISTER_TRACE(sched_process_exit);
- GATOR_UNREGISTER_TRACE(sched_process_wait);
- GATOR_UNREGISTER_TRACE(sched_process_fork);
pr_debug("gator: unregistered tracepoints\n");
for_each_present_cpu(cpu) {
@@ -216,17 +189,16 @@ void gator_trace_sched_stop(void)
kfree(per_cpu(theSchedBuf, cpu)[1]);
per_cpu(theSchedBuf, cpu)[0] = NULL;
per_cpu(theSchedBuf, cpu)[1] = NULL;
+ kfree(per_cpu(taskname_keys, cpu));
}
}
-int gator_trace_sched_read(int **buffer)
+int gator_trace_sched_read(long long **buffer)
{
- uint64_t time = gator_get_time();
int cpu = smp_processor_id();
unsigned long flags;
- int *schedBuf;
+ uint64_t *schedBuf;
int schedPos;
- int i;
if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)])
return 0;
@@ -242,20 +214,6 @@ int gator_trace_sched_read(int **buffer)
local_irq_restore(flags);
- // find mm and replace with cookies
- for (i = 0; i < schedPos; i += 6) {
- uint32_t cookie = schedBuf[i+3];
- if (cookie) {
- struct task_struct *task = (struct task_struct *)cookie;
- schedBuf[i+3] = get_exec_cookie(cpu, task);
- }
- }
-
- // timer/end event
- schedBuf[schedPos++] = SCHED_TIMER_EVENT;
- schedBuf[schedPos++] = (int)time;
- schedBuf[schedPos++] = (int)(time >> 32);
-
if (buffer)
*buffer = schedBuf;
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-18 07:50:53 +0000