gator: Version 5.155.15

Signed-off-by: Drew Richardson <drew.richardson@arm.com>

9 files changed, 278 insertions, 263 deletions

diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c
index 94f01e6..0670d6c 100644
--- a/driver/gator_backtrace.c
+++ b/driver/gator_backtrace.c
@@ -10,16 +10,22 @@
 /*
  * EABI backtrace stores {fp,lr} on the stack.
  */
-struct frame_tail_eabi {
+struct stack_frame_eabi {
 	union {
 		struct {
-			unsigned long fp;	// points to prev_lr
+			unsigned long fp;
+			// May be the fp in the case of a leaf function or clang
 			unsigned long lr;
+			// If lr is really the fp, lr2 is the corresponding lr
+			unsigned long lr2;
 		};
 		// Used to read 32 bit fp/lr from a 64 bit kernel
 		struct {
 			u32 fp_32;
+			// same as lr above
 			u32 lr_32;
+			// same as lr2 above
+			u32 lr2_32;
 		};
 	};
 };
@@ -27,61 +33,76 @@ struct frame_tail_eabi {
 static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int depth)
 {
 #if defined(__arm__) || defined(__aarch64__)
-	struct frame_tail_eabi *tail;
-	struct frame_tail_eabi *next;
-	struct frame_tail_eabi buftail;
+	struct stack_frame_eabi *curr;
+	struct stack_frame_eabi bufcurr;
 #if defined(__arm__)
 	const bool is_compat = false;
 	unsigned long fp = regs->ARM_fp;
 	unsigned long sp = regs->ARM_sp;
 	unsigned long lr = regs->ARM_lr;
-	const int frame_offset = 4;
+	const int gcc_frame_offset = sizeof(unsigned long);
 #else
 	// Is userspace aarch32 (32 bit)
 	const bool is_compat = compat_user_mode(regs);
 	unsigned long fp = (is_compat ? regs->regs[11] : regs->regs[29]);
 	unsigned long sp = (is_compat ? regs->compat_sp : regs->sp);
 	unsigned long lr = (is_compat ? regs->compat_lr : regs->regs[30]);
-	const int frame_offset = (is_compat ? 4 : 0);
+	const int gcc_frame_offset = (is_compat ? sizeof(u32) : 0);
 #endif
+	// clang frame offset is always zero
 	int is_user_mode = user_mode(regs);
 
+	// pc (current function) has already been added
+
 	if (!is_user_mode) {
 		return;
 	}
 
-	/* entry preamble may not have executed */
+	// Add the lr (parent function)
+	// entry preamble may not have executed
 	gator_add_trace(cpu, lr);
 
-	/* check tail is valid */
+	// check fp is valid
 	if (fp == 0 || fp < sp) {
 		return;
 	}
 
-	tail = (struct frame_tail_eabi *)(fp - frame_offset);
+	// Get the current stack frame
+	curr = (struct stack_frame_eabi *)(fp - gcc_frame_offset);
+	if ((unsigned long)curr & 3) {
+		return;
+	}
 
-	while (depth-- && tail && !((unsigned long)tail & 3)) {
-		/* Also check accessibility of one struct frame_tail beyond */
-		if (!access_ok(VERIFY_READ, tail, sizeof(struct frame_tail_eabi)))
-			return;
-		if (__copy_from_user_inatomic(&buftail, tail, sizeof(struct frame_tail_eabi)))
+	while (depth-- && curr) {
+		if (!access_ok(VERIFY_READ, curr, sizeof(struct stack_frame_eabi)) ||
+				__copy_from_user_inatomic(&bufcurr, curr, sizeof(struct stack_frame_eabi))) {
 			return;
+		}
+
+		fp = (is_compat ? bufcurr.fp_32 : bufcurr.fp);
+		lr = (is_compat ? bufcurr.lr_32 : bufcurr.lr);
+
+#define calc_next(reg) ((reg) - gcc_frame_offset)
+		// Returns true if reg is a valid fp
+#define validate_next(reg, curr) \
+		((reg) != 0 && (calc_next(reg) & 3) == 0 && (unsigned long)(curr) < calc_next(reg))
+
+		// Try lr from the stack as the fp because gcc leaf functions do not push lr
+		// If gcc_frame_offset is non-zero, the lr will also be the clang fp
+		// This assumes code is at a lower address than the stack
+		if (validate_next(lr, curr)) {
+			fp = lr;
+			lr = (is_compat ? bufcurr.lr2_32 : bufcurr.lr2);
+		}
 
-		lr = (is_compat ? buftail.lr_32 : buftail.lr);
 		gator_add_trace(cpu, lr);
 
-		/* frame pointers should progress back up the stack, towards higher addresses */
-		next = (struct frame_tail_eabi *)(lr - frame_offset);
-		if (tail >= next || lr == 0) {
-			fp = (is_compat ? buftail.fp_32 : buftail.fp);
-			next = (struct frame_tail_eabi *)(fp - frame_offset);
-			/* check tail is valid */
-			if (tail >= next || fp == 0) {
-				return;
-			}
+		if (!validate_next(fp, curr)) {
+			return;
 		}
 
-		tail = next;
+		// Move to the next stack frame
+		curr = (struct stack_frame_eabi *)calc_next(fp);
 	}
 #endif
 }
@@ -89,11 +110,12 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int
 #if defined(__arm__) || defined(__aarch64__)
 static int report_trace(struct stackframe *frame, void *d)
 {
-	unsigned int *depth = d, cookie = NO_COOKIE, cpu = get_physical_cpu();
+	unsigned int *depth = d, cookie = NO_COOKIE;
 	unsigned long addr = frame->pc;
 
 	if (*depth) {
 #if defined(MODULE)
+		unsigned int cpu = get_physical_cpu();
 		struct module *mod = __module_address(addr);
 		if (mod) {
 			cookie = get_cookie(cpu, current, mod->name, false);
diff --git a/driver/gator_events_l2c-310.c b/driver/gator_events_l2c-310.c
index 52472c7..21aa4a2 100644
--- a/driver/gator_events_l2c-310.c
+++ b/driver/gator_events_l2c-310.c
@@ -11,6 +11,10 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/module.h>
+#if defined(CONFIG_OF)
+#include <linux/of.h>
+#include <linux/of_address.h>
+#endif
 #include <asm/hardware/cache-l2x0.h>
 
 #include "gator.h"
@@ -150,10 +154,23 @@ static void __iomem *gator_events_l2c310_probe(void)
 #endif
 	};
 	int i;
+	void __iomem *base;
+#if defined(CONFIG_OF)
+	struct device_node *node = of_find_all_nodes(NULL);
 
-	for (i = 0; i < ARRAY_SIZE(variants); i++) {
-		void __iomem *base = ioremap(variants[i], SZ_4K);
+	if (node) {
+		of_node_put(node);
 
+		node = of_find_compatible_node(NULL, NULL, "arm,pl310-cache");
+		base = of_iomap(node, 0);
+		of_node_put(node);
+
+		return base;
+	}
+#endif
+
+	for (i = 0; i < ARRAY_SIZE(variants); i++) {
+		base = ioremap(variants[i], SZ_4K);
 		if (base) {
 			u32 cache_id = readl(base + L2X0_CACHE_ID);
 
diff --git a/driver/gator_events_mali_t6xx_hw.c b/driver/gator_events_mali_t6xx_hw.c
index fb2e15c..4f49c1d 100644
--- a/driver/gator_events_mali_t6xx_hw.c
+++ b/driver/gator_events_mali_t6xx_hw.c
@@ -22,14 +22,30 @@
 
 #include "gator_events_mali_common.h"
 
+/* If API version is not specified then assume API version 1. */
+#ifndef MALI_DDK_GATOR_API_VERSION
+#define MALI_DDK_GATOR_API_VERSION 1
+#endif
+
+#if (MALI_DDK_GATOR_API_VERSION != 1) && (MALI_DDK_GATOR_API_VERSION != 2)
+#error MALI_DDK_GATOR_API_VERSION is invalid (must be 1 for r1/r2 DDK, or 2 for r3 DDK).
+#endif
+
 /*
  * Mali-T6xx
  */
 typedef struct kbase_device *kbase_find_device_type(int);
 typedef kbase_context *kbase_create_context_type(kbase_device *);
 typedef void kbase_destroy_context_type(kbase_context *);
+
+#if MALI_DDK_GATOR_API_VERSION == 1
 typedef void *kbase_va_alloc_type(kbase_context *, u32);
 typedef void kbase_va_free_type(kbase_context *, void *);
+#elif MALI_DDK_GATOR_API_VERSION == 2
+typedef void *kbase_va_alloc_type(kbase_context *, u32, kbase_hwc_dma_mapping * handle);
+typedef void kbase_va_free_type(kbase_context *, kbase_hwc_dma_mapping * handle);
+#endif
+
 typedef mali_error kbase_instr_hwcnt_enable_type(kbase_context *, kbase_uk_hwcnt_setup *);
 typedef mali_error kbase_instr_hwcnt_disable_type(kbase_context *);
 typedef mali_error kbase_instr_hwcnt_clear_type(kbase_context *);
@@ -358,6 +374,11 @@ static const char *const hardware_counter_names[] = {
 /* Memory to dump hardware counters into */
 static void *kernel_dump_buffer;
 
+#if MALI_DDK_GATOR_API_VERSION == 2
+/* DMA state used to manage lifetime of the buffer */
+kbase_hwc_dma_mapping kernel_dump_buffer_handle;
+#endif
+
 /* kbase context and device */
 static kbase_context *kbcontext = NULL;
 static struct kbase_device *kbdevice = NULL;
@@ -524,9 +545,14 @@ static int start(void)
 		 *             * number of blocks (always 8 for midgard)
 		 *             * number of counters per block (always 64 for midgard)
 		 *             * number of bytes per counter (always 4 in midgard)
-		 * For a Mali-T6xx with a single core group = 1 * 8 * 64 * 4
+		 * For a Mali-T6xx with a single core group = 1 * 8 * 64 * 4 = 2048
+		 * For a Mali-T6xx with a dual core group   = 2 * 8 * 64 * 4 = 4096
 		 */
-		kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 2048);
+#if MALI_DDK_GATOR_API_VERSION == 1
+		kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096);
+#elif MALI_DDK_GATOR_API_VERSION == 2
+		kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096, &kernel_dump_buffer_handle);
+#endif
 		if (!kernel_dump_buffer) {
 			pr_debug("gator: Mali-T6xx: error trying to allocate va\n");
 			goto destroy_context;
@@ -556,7 +582,11 @@ static int start(void)
 	return 0;
 
 free_buffer:
+#if MALI_DDK_GATOR_API_VERSION == 1
 	kbase_va_free_symbol(kbcontext, kernel_dump_buffer);
+#elif MALI_DDK_GATOR_API_VERSION == 2
+	kbase_va_free_symbol(kbcontext, &kernel_dump_buffer_handle);
+#endif
 
 destroy_context:
 	kbase_destroy_context_symbol(kbcontext);
@@ -588,7 +618,13 @@ static void stop(void)
 		kbcontext = NULL;
 
 		kbase_instr_hwcnt_disable_symbol(temp_kbcontext);
+
+#if MALI_DDK_GATOR_API_VERSION == 1
 		kbase_va_free_symbol(temp_kbcontext, kernel_dump_buffer);
+#elif MALI_DDK_GATOR_API_VERSION == 2
+		kbase_va_free_symbol(temp_kbcontext, &kernel_dump_buffer_handle);
+#endif
+
 		kbase_destroy_context_symbol(temp_kbcontext);
 
 		pr_debug("gator: Mali-T6xx: hardware counters stopped\n");
diff --git a/driver/gator_events_perf_pmu.c b/driver/gator_events_perf_pmu.c
index 34a6bc7..53b2d0a 100644
--- a/driver/gator_events_perf_pmu.c
+++ b/driver/gator_events_perf_pmu.c
@@ -15,23 +15,36 @@
 
 extern bool event_based_sampling;
 
+// Maximum number of per-core counters - currently reserves enough space for two full hardware PMUs for big.LITTLE
 #define CNTMAX 16
 #define CCI_400 4
+// Maximum number of uncore counters
 // + 1 for the cci-400 cycles counter
 #define UCCNT (CCI_400 + 1)
 
+// A gator_attr is needed for every counter
 struct gator_attr {
+	// Set once in gator_events_perf_pmu_*_init - the name of the event in the gatorfs
 	char name[40];
+	// Exposed in gatorfs - set by gatord to enable this counter
 	unsigned long enabled;
+	// Set once in gator_events_perf_pmu_*_init - the perf type to use, see perf_type_id in the perf_event.h header file.
 	unsigned long type;
+	// Exposed in gatorfs - set by gatord to select the event to collect
 	unsigned long event;
+	// Exposed in gatorfs - set by gatord with the sample period to use and enable EBS for this counter
 	unsigned long count;
+	// Exposed as read only in gatorfs - set once in __attr_init as the key to use in the APC data
 	unsigned long key;
 };
 
+// Per-core counter attributes
 static struct gator_attr attrs[CNTMAX];
+// Number of initialized per-core counters
 static int attr_count;
+// Uncore counter attributes
 static struct gator_attr uc_attrs[UCCNT];
+// Number of initialized uncore counters
 static int uc_attr_count;
 
 struct gator_event {
@@ -458,7 +471,7 @@ int gator_events_perf_pmu_init(void)
 				break;
 			}
 		}
-		// Assume that valid PMUs are contigious
+		// Assume that valid PMUs are contiguous
 		if (IS_ERR(pe)) {
 			break;
 		}
@@ -470,6 +483,7 @@ int gator_events_perf_pmu_init(void)
 				found_cpu = true;
 				gator_events_perf_pmu_cpu_init(gator_cpu, type);
 			}
+			// Initialize gator_attrs for dynamic PMUs here
 		}
 
 		perf_event_release_kernel(pe);
@@ -483,6 +497,8 @@ int gator_events_perf_pmu_init(void)
 		gator_events_perf_pmu_cpu_init(gator_cpu, PERF_TYPE_RAW);
 	}
 
+	// Initialize gator_attrs for non-dynamic PMUs here
+
 	if (attr_count > CNTMAX) {
 		printk(KERN_ERR "gator: Too many perf counters\n");
 		return -1;
diff --git a/driver/gator_main.c b/driver/gator_main.c
index 3e62b59..46fe31d 100644
--- a/driver/gator_main.c
+++ b/driver/gator_main.c
@@ -8,7 +8,7 @@
  */
 
 // This version must match the gator daemon version
-static unsigned long gator_protocol_version = 13;
+static unsigned long gator_protocol_version = 14;
 
 #include <linux/slab.h>
 #include <linux/cpu.h>
@@ -162,8 +162,8 @@ static void gator_commit_buffer(int cpu, int buftype, u64 time);
 static int buffer_bytes_available(int cpu, int buftype);
 static bool buffer_check_space(int cpu, int buftype, int bytes);
 static int contiguous_space_available(int cpu, int bufytpe);
-static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
-static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
+static void gator_buffer_write_packed_int(int cpu, int buftype, int x);
+static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x);
 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
 static void gator_add_trace(int cpu, unsigned long address);
@@ -389,6 +389,8 @@ static bool buffer_commit_ready(int *cpu, int *buftype)
 				return true;
 			}
 	}
+	*cpu = -1;
+	*buftype = -1;
 	return false;
 }
 
@@ -476,14 +478,14 @@ static void gator_commit_buffer(int cpu, int buftype, u64 time)
 	if (length < 0) {
 		length += gator_buffer_size[buftype];
 	}
-	length = length - type_length - sizeof(int);
+	length = length - type_length - sizeof(s32);
 
 	if (length <= FRAME_HEADER_SIZE) {
 		// Nothing to write, only the frame header is present
 		return;
 	}
 
-	for (byte = 0; byte < sizeof(int); byte++) {
+	for (byte = 0; byte < sizeof(s32); byte++) {
 		per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
 	}
 
@@ -595,7 +597,7 @@ static void gator_timer_offline(void *migrate)
 
 	// Offline any events and output counters
 	time = gator_get_time();
-	if (marshal_event_header()) {
+	if (marshal_event_header(time)) {
 		list_for_each_entry(gi, &gator_events, list) {
 			if (gi->offline) {
 				len = gi->offline(&buffer, migrate);
@@ -644,11 +646,13 @@ static void gator_timer_online(void *migrate)
 	struct gator_interface *gi;
 	int len, cpu = get_physical_cpu();
 	int *buffer;
+	u64 time;
 
 	gator_trace_power_online();
 
 	// online any events and output counters
-	if (marshal_event_header()) {
+	time = gator_get_time();
+	if (marshal_event_header(time)) {
 		list_for_each_entry(gi, &gator_events, list) {
 			if (gi->online) {
 				len = gi->online(&buffer, migrate);
@@ -656,7 +660,7 @@ static void gator_timer_online(void *migrate)
 			}
 		}
 		// Only check after writing all counters so that time and corresponding counters appear in the same frame
-		buffer_check(cpu, BLOCK_COUNTER_BUF, gator_get_time());
+		buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 	}
 
 	if (!migrate) {
@@ -745,8 +749,6 @@ static u64 gator_get_time(void)
 		if (!printed_monotonic_warning && delta > 500000) {
 			printk(KERN_ERR "%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
 			printed_monotonic_warning = true;
-		} else {
-			pr_debug("%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\n", __FUNCTION__, cpu, delta);
 		}
 		timestamp = prev_timestamp;
 	}
@@ -894,6 +896,7 @@ static int gator_init(void)
 		if (gator_events_list[i])
 			gator_events_list[i]();
 
+	gator_trace_sched_init();
 	gator_trace_power_init();
 
 	return 0;
@@ -1227,83 +1230,81 @@ static int userspace_buffer_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
-				     size_t count, loff_t *offset)
+static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
 {
-	int retval = -EINVAL;
-	int commit = 0, length1, length2, read;
+	int commit, length1, length2, read;
 	char *buffer1;
-	char *buffer2 = NULL;
+	char *buffer2;
 	int cpu, buftype;
+	int written = 0;
 
-	/* do not handle partial reads */
-	if (count != userspace_buffer_size || *offset)
+	// ensure there is enough space for a whole frame
+	if (count < userspace_buffer_size || *offset) {
 		return -EINVAL;
+	}
 
 	// sleep until the condition is true or a signal is received
 	// the condition is checked each time gator_buffer_wait is woken up
-	buftype = cpu = -1;
 	wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
 
-	if (signal_pending(current))
+	if (signal_pending(current)) {
 		return -EINTR;
-
-	length2 = 0;
-	retval = -EFAULT;
-
-	mutex_lock(&gator_buffer_mutex);
+	}
 
 	if (buftype == -1 || cpu == -1) {
-		retval = 0;
-		goto out;
+		return 0;
 	}
 
-	read = per_cpu(gator_buffer_read, cpu)[buftype];
-	commit = per_cpu(gator_buffer_commit, cpu)[buftype];
+	mutex_lock(&gator_buffer_mutex);
 
-	/* May happen if the buffer is freed during pending reads. */
-	if (!per_cpu(gator_buffer, cpu)[buftype]) {
-		retval = -EFAULT;
-		goto out;
-	}
+	do {
+		read = per_cpu(gator_buffer_read, cpu)[buftype];
+		commit = per_cpu(gator_buffer_commit, cpu)[buftype];
 
-	/* determine the size of two halves */
-	length1 = commit - read;
-	buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
-	buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
-	if (length1 < 0) {
-		length1 = gator_buffer_size[buftype] - read;
-		length2 = commit;
-	}
+		// May happen if the buffer is freed during pending reads.
+		if (!per_cpu(gator_buffer, cpu)[buftype]) {
+			break;
+		}
 
-	/* start, middle or end */
-	if (length1 > 0) {
-		if (copy_to_user(&buf[0], buffer1, length1)) {
-			goto out;
+		// determine the size of two halves
+		length1 = commit - read;
+		length2 = 0;
+		buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
+		buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
+		if (length1 < 0) {
+			length1 = gator_buffer_size[buftype] - read;
+			length2 = commit;
 		}
-	}
 
-	/* possible wrap around */
-	if (length2 > 0) {
-		if (copy_to_user(&buf[length1], buffer2, length2)) {
-			goto out;
+		if (length1 + length2 > count - written) {
+			break;
 		}
-	}
 
-	per_cpu(gator_buffer_read, cpu)[buftype] = commit;
-	retval = length1 + length2;
+		// start, middle or end
+		if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) {
+			break;
+		}
 
-	/* kick just in case we've lost an SMP event */
-	wake_up(&gator_buffer_wait);
+		// possible wrap around
+		if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) {
+			break;
+		}
 
-	// Wake up annotate_write if more space is available
-	if (buftype == ANNOTATE_BUF) {
-		wake_up(&gator_annotate_wait);
-	}
+		per_cpu(gator_buffer_read, cpu)[buftype] = commit;
+		written += length1 + length2;
+
+		// Wake up annotate_write if more space is available
+		if (buftype == ANNOTATE_BUF) {
+			wake_up(&gator_annotate_wait);
+		}
+	}	while (buffer_commit_ready(&cpu, &buftype));
 
-out:
 	mutex_unlock(&gator_buffer_mutex);
-	return retval;
+
+	// kick just in case we've lost an SMP event
+	wake_up(&gator_buffer_wait);
+
+	return written > 0 ? written : -EFAULT;
 }
 
 const struct file_operations gator_event_buffer_fops = {
@@ -1376,6 +1377,9 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root)
 		if (gi->create_files)
 			gi->create_files(sb, dir);
 
+	// Sched Events
+	sched_trace_create_files(sb, dir);
+
 	// Power interface
 	gator_trace_power_create_files(sb, dir);
 }
diff --git a/driver/gator_marshaling.c b/driver/gator_marshaling.c
index 627b441..72dd5ef 100644
--- a/driver/gator_marshaling.c
+++ b/driver/gator_marshaling.c
@@ -28,7 +28,13 @@ static void marshal_summary(long long timestamp, long long uptime, const char *
 	gator_buffer_write_string(cpu, SUMMARY_BUF, NEWLINE_CANARY);
 	gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, timestamp);
 	gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, uptime);
+	gator_buffer_write_string(cpu, SUMMARY_BUF, "uname");
 	gator_buffer_write_string(cpu, SUMMARY_BUF, uname);
+#if GATOR_IKS_SUPPORT
+	gator_buffer_write_string(cpu, SUMMARY_BUF, "iks");
+	gator_buffer_write_string(cpu, SUMMARY_BUF, "");
+#endif
+	gator_buffer_write_string(cpu, SUMMARY_BUF, "");
 	// Commit the buffer now so it can be one of the first frames read by Streamline
 	gator_commit_buffer(cpu, SUMMARY_BUF, gator_get_time());
 	local_irq_restore(flags);
@@ -102,7 +108,7 @@ static void marshal_backtrace_footer(void)
 	buffer_check(cpu, BACKTRACE_BUF, gator_get_time());
 }
 
-static bool marshal_event_header(void)
+static bool marshal_event_header(u64 time)
 {
 	unsigned long flags, cpu = get_physical_cpu();
 	bool retval = false;
@@ -110,7 +116,7 @@ static bool marshal_event_header(void)
 	local_irq_save(flags);
 	if (buffer_check_space(cpu, BLOCK_COUNTER_BUF, MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
 		gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, 0);	// key of zero indicates a timestamp
-		gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, gator_get_time());
+		gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, time);
 		retval = true;
 	}
 	local_irq_restore(flags);
@@ -343,7 +349,7 @@ static void marshal_frame(int cpu, int buftype)
 	}
 
 	// leave space for 4-byte unpacked length
-	per_cpu(gator_buffer_write, cpu)[buftype] = (per_cpu(gator_buffer_write, cpu)[buftype] + 4) & gator_buffer_mask[buftype];
+	per_cpu(gator_buffer_write, cpu)[buftype] = (per_cpu(gator_buffer_write, cpu)[buftype] + sizeof(s32)) & gator_buffer_mask[buftype];
 
 	// add frame type and core number
 	gator_buffer_write_packed_int(cpu, buftype, frame);
diff --git a/driver/gator_pack.c b/driver/gator_pack.c
index 2bddcbe..2c082f2 100644
--- a/driver/gator_pack.c
+++ b/driver/gator_pack.c
@@ -7,179 +7,52 @@
  *
  */
 
-static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x)
+static void gator_buffer_write_packed_int(int cpu, int buftype, int x)
 {
 	uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype];
 	uint32_t mask = gator_buffer_mask[buftype];
 	char *buffer = per_cpu(gator_buffer, cpu)[buftype];
-	int write0 = (write + 0) & mask;
-	int write1 = (write + 1) & mask;
+	int packedBytes = 0;
+	int more = true;
+	while (more) {
+		// low order 7 bits of x
+		char b = x & 0x7f;
+		x >>= 7;
 
-	if ((x & 0xffffff80) == 0) {
-		buffer[write0] = x & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write1;
-	} else if ((x & 0xffffc000) == 0) {
-		int write2 = (write + 2) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write2;
-	} else if ((x & 0xffe00000) == 0) {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write3;
-	} else if ((x & 0xf0000000) == 0) {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		int write4 = (write + 4) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) | 0x80;
-		buffer[write3] = (x >> 21) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write4;
-	} else {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		int write4 = (write + 4) & mask;
-		int write5 = (write + 5) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) | 0x80;
-		buffer[write3] = (x >> 21) | 0x80;
-		buffer[write4] = (x >> 28) & 0x0f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write5;
+		if ((x == 0 && (b & 0x40) == 0) || (x == -1 && (b & 0x40) != 0)) {
+			more = false;
+		} else {
+			b |= 0x80;
+		}
+
+		buffer[(write + packedBytes) & mask] = b;
+		packedBytes++;
 	}
+
+	per_cpu(gator_buffer_write, cpu)[buftype] = (write + packedBytes) & mask;
 }
 
-static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x)
+static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x)
 {
 	uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype];
 	uint32_t mask = gator_buffer_mask[buftype];
 	char *buffer = per_cpu(gator_buffer, cpu)[buftype];
-	int write0 = (write + 0) & mask;
-	int write1 = (write + 1) & mask;
+	int packedBytes = 0;
+	int more = true;
+	while (more) {
+		// low order 7 bits of x
+		char b = x & 0x7f;
+		x >>= 7;
 
-	if ((x & 0xffffffffffffff80LL) == 0) {
-		buffer[write0] = x & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write1;
-	} else if ((x & 0xffffffffffffc000LL) == 0) {
-		int write2 = (write + 2) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write2;
-	} else if ((x & 0xffffffffffe00000LL) == 0) {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write3;
-	} else if ((x & 0xfffffffff0000000LL) == 0) {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		int write4 = (write + 4) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) | 0x80;
-		buffer[write3] = (x >> 21) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write4;
-	} else if ((x & 0xfffffff800000000LL) == 0) {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		int write4 = (write + 4) & mask;
-		int write5 = (write + 5) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) | 0x80;
-		buffer[write3] = (x >> 21) | 0x80;
-		buffer[write4] = (x >> 28) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write5;
-	} else if ((x & 0xfffffc0000000000LL) == 0) {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		int write4 = (write + 4) & mask;
-		int write5 = (write + 5) & mask;
-		int write6 = (write + 6) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) | 0x80;
-		buffer[write3] = (x >> 21) | 0x80;
-		buffer[write4] = (x >> 28) | 0x80;
-		buffer[write5] = (x >> 35) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write6;
-	} else if ((x & 0xfffe000000000000LL) == 0) {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		int write4 = (write + 4) & mask;
-		int write5 = (write + 5) & mask;
-		int write6 = (write + 6) & mask;
-		int write7 = (write + 7) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) | 0x80;
-		buffer[write3] = (x >> 21) | 0x80;
-		buffer[write4] = (x >> 28) | 0x80;
-		buffer[write5] = (x >> 35) | 0x80;
-		buffer[write6] = (x >> 42) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write7;
-	} else if ((x & 0xff00000000000000LL) == 0) {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		int write4 = (write + 4) & mask;
-		int write5 = (write + 5) & mask;
-		int write6 = (write + 6) & mask;
-		int write7 = (write + 7) & mask;
-		int write8 = (write + 8) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) | 0x80;
-		buffer[write3] = (x >> 21) | 0x80;
-		buffer[write4] = (x >> 28) | 0x80;
-		buffer[write5] = (x >> 35) | 0x80;
-		buffer[write6] = (x >> 42) | 0x80;
-		buffer[write7] = (x >> 49) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write8;
-	} else if ((x & 0x8000000000000000LL) == 0) {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		int write4 = (write + 4) & mask;
-		int write5 = (write + 5) & mask;
-		int write6 = (write + 6) & mask;
-		int write7 = (write + 7) & mask;
-		int write8 = (write + 8) & mask;
-		int write9 = (write + 9) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) | 0x80;
-		buffer[write3] = (x >> 21) | 0x80;
-		buffer[write4] = (x >> 28) | 0x80;
-		buffer[write5] = (x >> 35) | 0x80;
-		buffer[write6] = (x >> 42) | 0x80;
-		buffer[write7] = (x >> 49) | 0x80;
-		buffer[write8] = (x >> 56) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write9;
-	} else {
-		int write2 = (write + 2) & mask;
-		int write3 = (write + 3) & mask;
-		int write4 = (write + 4) & mask;
-		int write5 = (write + 5) & mask;
-		int write6 = (write + 6) & mask;
-		int write7 = (write + 7) & mask;
-		int write8 = (write + 8) & mask;
-		int write9 = (write + 9) & mask;
-		int write10 = (write + 10) & mask;
-		buffer[write0] = x | 0x80;
-		buffer[write1] = (x >> 7) | 0x80;
-		buffer[write2] = (x >> 14) | 0x80;
-		buffer[write3] = (x >> 21) | 0x80;
-		buffer[write4] = (x >> 28) | 0x80;
-		buffer[write5] = (x >> 35) | 0x80;
-		buffer[write6] = (x >> 42) | 0x80;
-		buffer[write7] = (x >> 49) | 0x80;
-		buffer[write8] = (x >> 56) | 0x80;
-		buffer[write9] = (x >> 63) & 0x7f;
-		per_cpu(gator_buffer_write, cpu)[buftype] = write10;
+		if ((x == 0 && (b & 0x40) == 0) || (x == -1 && (b & 0x40) != 0)) {
+			more = false;
+		} else {
+			b |= 0x80;
+		}
+
+		buffer[(write + packedBytes) & mask] = b;
+		packedBytes++;
 	}
+
+	per_cpu(gator_buffer_write, cpu)[buftype] = (write + packedBytes) & mask;
 }
diff --git a/driver/gator_trace_gpu.c b/driver/gator_trace_gpu.c
index c94f6a0..61ecbe3 100644
--- a/driver/gator_trace_gpu.c
+++ b/driver/gator_trace_gpu.c
@@ -82,9 +82,12 @@ static void mali_gpu_stop(int unit, int core)
 	int last_pid = 0;
 
 	spin_lock(&mali_gpu_jobs_lock);
+	if (mali_gpu_jobs[unit][core].count == 0) {
+		spin_unlock(&mali_gpu_jobs_lock);
+		return;
+	}
 	--mali_gpu_jobs[unit][core].count;
 	count = mali_gpu_jobs[unit][core].count;
-	BUG_ON(count < 0);
 	if (count) {
 		last_core = mali_gpu_jobs[unit][core].last_core;
 		last_tgid = mali_gpu_jobs[unit][core].last_tgid;
diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c
index eb989b5..e989f6a 100644
--- a/driver/gator_trace_sched.c
+++ b/driver/gator_trace_sched.c
@@ -16,15 +16,44 @@
 #define TASK_MAP_ENTRIES		1024	/* must be power of 2 */
 #define TASK_MAX_COLLISIONS		2
 
-static DEFINE_PER_CPU(uint64_t *, taskname_keys);
-static DEFINE_PER_CPU(int, collecting);
-
 enum {
 	STATE_WAIT_ON_OTHER = 0,
 	STATE_CONTENTION,
 	STATE_WAIT_ON_IO,
+	CPU_WAIT_TOTAL
 };
 
+static DEFINE_PER_CPU(uint64_t *, taskname_keys);
+static DEFINE_PER_CPU(int, collecting);
+
+// this array is never read as the cpu wait charts are derived counters
+// the files are needed, nonetheless, to show that these counters are available
+static ulong cpu_wait_enabled[CPU_WAIT_TOTAL];
+static ulong sched_cpu_key[CPU_WAIT_TOTAL];
+
+static int sched_trace_create_files(struct super_block *sb, struct dentry *root)
+{
+	struct dentry *dir;
+
+	// CPU Wait - Contention
+	dir = gatorfs_mkdir(sb, root, "Linux_cpu_wait_contention");
+	if (!dir) {
+		return -1;
+	}
+	gatorfs_create_ulong(sb, dir, "enabled", &cpu_wait_enabled[STATE_CONTENTION]);
+	gatorfs_create_ro_ulong(sb, dir, "key", &sched_cpu_key[STATE_CONTENTION]);
+
+	// CPU Wait - I/O
+	dir = gatorfs_mkdir(sb, root, "Linux_cpu_wait_io");
+	if (!dir) {
+		return -1;
+	}
+	gatorfs_create_ulong(sb, dir, "enabled", &cpu_wait_enabled[STATE_WAIT_ON_IO]);
+	gatorfs_create_ro_ulong(sb, dir, "key", &sched_cpu_key[STATE_WAIT_ON_IO]);
+
+	return 0;
+}
+
 void emit_pid_name(struct task_struct *task)
 {
 	bool found = false;
@@ -70,7 +99,8 @@ static void collect_counters(void)
 	struct gator_interface *gi;
 	u64 time;
 
-	if (marshal_event_header()) {
+	time = gator_get_time();
+	if (marshal_event_header(time)) {
 		list_for_each_entry(gi, &gator_events, list) {
 			if (gi->read) {
 				len = gi->read(&buffer);
@@ -81,7 +111,6 @@ static void collect_counters(void)
 			}
 		}
 		// Only check after writing all counters so that time and corresponding counters appear in the same frame
-		time = gator_get_time();
 		buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 
 #if GATOR_LIVE
@@ -216,3 +245,12 @@ void gator_trace_sched_stop(void)
 		kfree(per_cpu(taskname_keys, cpu));
 	}
 }
+
+void gator_trace_sched_init(void)
+{
+	int i;
+	for (i = 0; i < CPU_WAIT_TOTAL; i++) {
+		cpu_wait_enabled[i] = 0;
+		sched_cpu_key[i] = gator_events_get_key();
+	}
+}