diff options
| -rw-r--r-- | arch/arm/mach-omap2/pm34xx.c | 18 | ||||
| -rw-r--r-- | arch/arm/mach-omap2/trace-clock.c | 310 | ||||
| -rw-r--r-- | arch/arm/plat-omap/include/plat/trace-clock.h | 4 | ||||
| -rw-r--r-- | kernel/trace/trace-clock-32-to-64.c | 21 |
4 files changed, 287 insertions, 66 deletions
diff --git a/arch/arm/mach-omap2/pm34xx.c b/arch/arm/mach-omap2/pm34xx.c index d5a3dbd2a63..dcb1dd36c24 100644 --- a/arch/arm/mach-omap2/pm34xx.c +++ b/arch/arm/mach-omap2/pm34xx.c @@ -528,24 +528,20 @@ static void omap3_pm_idle(void) goto out; trace_pm_idle_entry(); - /* - * Should only be stopped when the CPU is stopping the ccnt - * counter in idle. sleep_while_idle seems to disable - * the ccnt clock (as of 2.6.32-rc8). - */ - stop_trace_clock(); + save_sync_trace_clock(); omap_sram_idle(); /* - * Restarting the trace clock should ideally be done much sooner. When + * Resyncing the trace clock should ideally be done much sooner. When * we arrive here, there are already some interrupt handlers which have * run before us, using potentially wrong timestamps. This leads * to problems when restarting the clock (and synchronizing on the 32k * clock) if the cycle counter was still active. - * start_track_clock must ensure that timestamps never ever go backward. + * resync_track_clock must ensure that timestamps never ever go + * backward. */ - start_trace_clock(); + resync_trace_clock(); trace_pm_idle_exit(); out: @@ -578,9 +574,9 @@ static int omap3_pm_suspend(void) omap3_intc_suspend(); trace_pm_suspend_entry(); - stop_trace_clock(); + save_sync_trace_clock(); omap_sram_idle(); - start_trace_clock(); + resync_trace_clock(); trace_pm_suspend_exit(); restore: diff --git a/arch/arm/mach-omap2/trace-clock.c b/arch/arm/mach-omap2/trace-clock.c index 154bf40e061..32a6bd0d09e 100644 --- a/arch/arm/mach-omap2/trace-clock.c +++ b/arch/arm/mach-omap2/trace-clock.c @@ -2,9 +2,8 @@ * arch/arm/mach-omap2/trace-clock.c * * Trace clock for ARM OMAP3 - * Currently uniprocessor-only. * - * Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>, February 2009 + * Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> 2009 */ #include <linux/module.h> @@ -12,6 +11,8 @@ #include <linux/timer.h> #include <linux/spinlock.h> #include <linux/init.h> +#include <linux/cpu.h> + #include <plat/clock.h> #include <plat/trace-clock.h> @@ -19,13 +20,25 @@ /* Need direct access to the clock from arch/arm/mach-omap2/timer-gp.c */ static struct clocksource *clock; -/* 32KHz counter count save upon PM sleep */ -static u32 saved_32k_count; -static u64 saved_trace_clock; +/* 32KHz counter per-cpu count save upon PM sleep */ +struct pm_save_count { + u64 int_fast_clock; + struct timer_list clear_ccnt_ms_timer; + u32 ext_32k; + int refcount; + u32 init_clock; + spinlock_t lock; /* spinlock only sync the refcount */ + /* + * Is fast clock ready to be read ? Read with preemption off. Modified + * only by local CPU in thread and interrupt context or by start/stop + * when time is not read concurrently. + */ + int fast_clock_ready; +}; -static void clear_ccnt_ms(unsigned long data); +static DEFINE_PER_CPU(struct pm_save_count, pm_save_count); -static DEFINE_TIMER(clear_ccnt_ms_timer, clear_ccnt_ms, 0, 0); +static void clear_ccnt_ms(unsigned long data); /* According to timer32k.c, this is a 32768Hz clock, not a 32000Hz clock. */ #define TIMER_32K_FREQ 32768 @@ -107,8 +120,10 @@ static inline void write_ccnt(u32 val) */ static void clear_ccnt_ms(unsigned long data) { + struct pm_save_count *pm_count; unsigned int cycles; unsigned long flags; + int cpu; local_irq_save(flags); isb(); /* clear the pipeline so we can execute ASAP */ @@ -120,15 +135,98 @@ static void clear_ccnt_ms(unsigned long data) isb(); local_irq_restore(flags); - mod_timer(&clear_ccnt_ms_timer, jiffies + clear_ccnt_interval); + cpu = smp_processor_id(); + pm_count = &per_cpu(pm_save_count, cpu); + pm_count->clear_ccnt_ms_timer.expires = jiffies + clear_ccnt_interval; + add_timer_on(&pm_count->clear_ccnt_ms_timer, cpu); } -void _start_trace_clock(void) +/* + * disabling interrupts to protect against concurrent IPI save/resync. + */ +void save_sync_trace_clock(void) { + struct pm_save_count *pm_count; + unsigned long flags; + int cpu; + + local_irq_save(flags); + cpu = smp_processor_id(); + pm_count = &per_cpu(pm_save_count, cpu); + spin_lock(&pm_count->lock); + + if (!pm_count->refcount) + goto end; + + pm_count->ext_32k = clock->read(clock); + pm_count->int_fast_clock = trace_clock_read64(); +end: + spin_unlock(&pm_count->lock); + + /* + * Only enable slow read after saving the clock values. + */ + barrier(); + pm_count->fast_clock_ready = 0; + local_irq_restore(flags); +} + +/* + * Called with preemption disabled. Read the external clock source directly + * and return corresponding time in fast clock source time frame. + * Only called after time is saved and before it is resynced. + */ +u64 _trace_clock_read_slow(void) +{ + struct pm_save_count *pm_count; + u64 ref_time; + unsigned int count_32k; + int cpu; + + cpu = smp_processor_id(); + pm_count = &per_cpu(pm_save_count, cpu); + WARN_ON_ONCE(pm_count->fast_clock_ready); + WARN_ON_ONCE(!pm_count->refcount); + + /* + * Set the timer's value MSBs to the same as current 32K timer. + */ + ref_time = pm_count->int_fast_clock; + if (!pm_count->init_clock) + count_32k = clock->read(clock); + else + count_32k = pm_count->init_clock; + + /* + * Delta done on 32-bits, then casted to u64. Must guarantee + * that we are called often enough so the difference does not + * overflow 32 bits anyway. + */ + ref_time += (u64)(count_32k - pm_count->ext_32k) + * (cpu_hz >> TIMER_32K_SHIFT); + return ref_time; +} +EXPORT_SYMBOL_GPL(_trace_clock_read_slow); + +/* + * resynchronize the per-cpu fast clock with the last save_sync values and the + * external clock. Called from PM (thread) context and IPI context. + */ +void resync_trace_clock(void) +{ + struct pm_save_count *pm_count; + u64 ref_time; unsigned long flags; - unsigned int count_32k, count_trace_clock; u32 regval; - u64 ref_time, prev_time; + int cpu; + + local_irq_save(flags); + cpu = smp_processor_id(); + pm_count = &per_cpu(pm_save_count, cpu); + spin_lock(&pm_count->lock); + + if (!pm_count->refcount) + goto end; /* Let userspace access performance counter registers */ regval = read_useren(); @@ -145,55 +243,114 @@ void _start_trace_clock(void) regval &= ~(1 << 5); /* Enable even in non-invasive debug prohib. */ write_pmnc(regval); - /* - * Set the timer's value MSBs to the same as current 32K timer. - */ - ref_time = saved_trace_clock; - local_irq_save(flags); - count_32k = clock->read(clock); - prev_time = trace_clock_read64(); - /* - * Delta done on 32-bits, then casted to u64. Must guarantee - * that we are called often enough so the difference does not - * overflow 32 bits anyway. - */ - ref_time += (u64)(count_32k - saved_32k_count) - * (cpu_hz >> TIMER_32K_SHIFT); - /* Make sure we never _ever_ decrement the clock value */ - if (ref_time < prev_time) - ref_time = prev_time; + ref_time = _trace_clock_read_slow(); + + if (pm_count->init_clock) + pm_count->init_clock = 0; + write_ctens(read_ctens() & ~(1 << 31)); /* disable counter */ write_ccnt((u32)ref_time & ~(1 << 31)); write_ctens(read_ctens() | (1 << 31)); /* enable counter */ - count_trace_clock = trace_clock_read32(); - _trace_clock_write_synthetic_tsc(ref_time); - local_irq_restore(flags); - get_synthetic_tsc(); - - /* mod_timer generates a trace event. Must run after time-base update */ - mod_timer(&clear_ccnt_ms_timer, jiffies + clear_ccnt_interval); + _trace_clock_write_synthetic_tsc(ref_time); + barrier(); /* make clock ready before enabling */ + pm_count->fast_clock_ready = 1; - if (unlikely(!print_info_done || saved_trace_clock > ref_time)) { + if (unlikely(!print_info_done)) { printk(KERN_INFO "Trace clock using cycle counter at %llu HZ\n" - "32k clk value 0x%08X, cycle counter value 0x%08X\n" "saved 32k clk value 0x%08X, " "saved cycle counter value 0x%016llX\n" "synthetic value (write, read) 0x%016llX, 0x%016llX\n", cpu_hz, - count_32k, count_trace_clock, - saved_32k_count, saved_trace_clock, + pm_count->ext_32k, + pm_count->int_fast_clock, ref_time, trace_clock_read64()); printk(KERN_INFO "Reference clock used : %s\n", clock->name); print_info_done = 1; } +end: + spin_unlock(&pm_count->lock); + local_irq_restore(flags); +} + +static void prepare_timer(int cpu) +{ + struct pm_save_count *pm_count; + + pm_count = &per_cpu(pm_save_count, cpu); + init_timer_deferrable(&pm_count->clear_ccnt_ms_timer); + pm_count->clear_ccnt_ms_timer.function = clear_ccnt_ms; + pm_count->clear_ccnt_ms_timer.expires = jiffies + clear_ccnt_interval; +} + +static void enable_timer(int cpu) +{ + struct pm_save_count *pm_count; + + pm_count = &per_cpu(pm_save_count, cpu); + add_timer_on(&pm_count->clear_ccnt_ms_timer, cpu); +} + +static void disable_timer_ipi(void *info) +{ + struct pm_save_count *pm_count; + int cpu = smp_processor_id(); + + pm_count = &per_cpu(pm_save_count, cpu); + del_timer(&pm_count->clear_ccnt_ms_timer); + save_sync_trace_clock(); +} + +static void disable_timer(int cpu) +{ + smp_call_function_single(cpu, disable_timer_ipi, NULL, 1); +} + +static void resync_ipi(void *info) +{ + resync_trace_clock(); +} + +void _start_trace_clock(void) +{ + struct pm_save_count *pm_count; + u32 ext_32k; + u64 old_fast_clock; + int cpu; + + ext_32k = clock->read(clock); + old_fast_clock = per_cpu(pm_save_count, 0).int_fast_clock; + + for_each_online_cpu(cpu) { + pm_count = &per_cpu(pm_save_count, cpu); + pm_count->ext_32k = ext_32k; + pm_count->int_fast_clock = old_fast_clock; + pm_count->refcount = 1; + pm_count->init_clock = ext_32k; + } + + on_each_cpu(resync_ipi, NULL, 1); + + get_synthetic_tsc(); + + for_each_online_cpu(cpu) { + prepare_timer(cpu); + enable_timer(cpu); + } } void _stop_trace_clock(void) { - saved_32k_count = clock->read(clock); - saved_trace_clock = trace_clock_read64(); - del_timer_sync(&clear_ccnt_ms_timer); + struct pm_save_count *pm_count; + int cpu; + + per_cpu(pm_save_count, 0).int_fast_clock = trace_clock_read64(); + + for_each_online_cpu(cpu) { + pm_count = &per_cpu(pm_save_count, cpu); + disable_timer(cpu); + pm_count->refcount = 0; + } put_synthetic_tsc(); } @@ -217,6 +374,71 @@ end: spin_unlock(&trace_clock_lock); } +/* + * hotcpu_callback - CPU hotplug callback + * @nb: notifier block + * @action: hotplug action to take + * @hcpu: CPU number + * + * Start/stop timers for trace clock upon cpu hotplug. + * Also resync the clock. + * + * Returns the success/failure of the operation. (NOTIFY_OK, NOTIFY_BAD) + */ +static int __cpuinit hotcpu_callback(struct notifier_block *nb, + unsigned long action, + void *hcpu) +{ + struct pm_save_count *pm_count; + unsigned int hotcpu = (unsigned long)hcpu; + unsigned long flags; + + switch (action) { + case CPU_UP_PREPARE: + case CPU_UP_PREPARE_FROZEN: + spin_lock(&trace_clock_lock); + if (trace_clock_refcount) { + pm_count = &per_cpu(pm_save_count, hotcpu); + local_irq_save(flags); + pm_count->ext_32k = clock->read(clock); + pm_count->int_fast_clock = trace_clock_read64(); + local_irq_restore(flags); + pm_count->refcount = 1; + prepare_timer(hotcpu); + } + spin_unlock(&trace_clock_lock); + break; + case CPU_ONLINE: + case CPU_ONLINE_FROZEN: + spin_lock(&trace_clock_lock); + if (trace_clock_refcount) { + resync_trace_clock(); + enable_timer(hotcpu); + } + spin_unlock(&trace_clock_lock); + break; +#ifdef CONFIG_HOTPLUG_CPU + case CPU_DOWN_PREPARE: + case CPU_DOWN_PREPARE_FROZEN: + spin_lock(&trace_clock_lock); + if (trace_clock_refcount) + disable_timer(hotcpu); + spin_unlock(&trace_clock_lock); + break; + case CPU_DEAD: + case CPU_DEAD_FROZEN: + spin_lock(&trace_clock_lock); + if (trace_clock_refcount) { + pm_count = &per_cpu(pm_save_count, hotcpu); + pm_count->refcount = 0; + } + spin_unlock(&trace_clock_lock); + break; +#endif /* CONFIG_HOTPLUG_CPU */ + } + return NOTIFY_OK; +} + void get_trace_clock(void) { spin_lock(&trace_clock_lock); @@ -243,6 +465,7 @@ EXPORT_SYMBOL_GPL(put_trace_clock); static __init int init_trace_clock(void) { + int cpu; u64 rem; clock = get_clocksource_32k(); @@ -250,6 +473,9 @@ static __init int init_trace_clock(void) cpu_hz, &rem); printk(KERN_INFO "LTTng will clear ccnt top bit every %u jiffies.\n", clear_ccnt_interval); + hotcpu_notifier(hotcpu_callback, 4); + for_each_possible_cpu(cpu) + spin_lock_init(&per_cpu(pm_save_count, cpu).lock); return 0; } __initcall(init_trace_clock); diff --git a/arch/arm/plat-omap/include/plat/trace-clock.h b/arch/arm/plat-omap/include/plat/trace-clock.h index 344938d2a47..f7afeb77ad0 100644 --- a/arch/arm/plat-omap/include/plat/trace-clock.h +++ b/arch/arm/plat-omap/include/plat/trace-clock.h @@ -83,9 +83,9 @@ extern void put_trace_clock(void); extern void get_synthetic_tsc(void); extern void put_synthetic_tsc(void); -/* Used by the architecture upon wakeup from PM idle */ +extern void resync_trace_clock(void); +extern void save_sync_trace_clock(void); extern void start_trace_clock(void); -/* Used by the architecture when going to PM idle */ extern void stop_trace_clock(void); static inline void set_trace_clock_is_sync(int state) diff --git a/kernel/trace/trace-clock-32-to-64.c b/kernel/trace/trace-clock-32-to-64.c index 1e0a9382ecc..d80255eb288 100644 --- a/kernel/trace/trace-clock-32-to-64.c +++ b/kernel/trace/trace-clock-32-to-64.c @@ -97,17 +97,18 @@ static void update_synthetic_tsc(void) } /* - * Should only be called when the synthetic clock is not used. + * Should only be called when interrupts are off. Affects only current CPU. */ void _trace_clock_write_synthetic_tsc(u64 value) { struct synthetic_tsc_struct *cpu_synth; - int cpu; + unsigned int new_index; - for_each_online_cpu(cpu) { - cpu_synth = &per_cpu(synthetic_tsc, cpu); - cpu_synth->tsc[cpu_synth->index].val = value; - } + cpu_synth = &per_cpu(synthetic_tsc, smp_processor_id()); + new_index = 1 - cpu_synth->index; /* 0 <-> 1 */ + cpu_synth->tsc[new_index].val = value; + barrier(); + cpu_synth->index = new_index; /* atomic change of index */ } /* Called from buffer switch : in _any_ context (even NMI) */ @@ -187,7 +188,7 @@ static void prepare_synthetic_tsc(int cpu) cpu_synth->tsc[0].val = local_count; cpu_synth->index = 0; smp_wmb(); /* Writing in data of CPU about to come up */ - init_timer(&per_cpu(tsc_timer, cpu)); + init_timer_deferrable(&per_cpu(tsc_timer, cpu)); per_cpu(tsc_timer, cpu).function = tsc_timer_fct; per_cpu(tsc_timer, cpu).expires = jiffies + precalc_expire; } @@ -245,10 +246,8 @@ static int __cpuinit hotcpu_callback(struct notifier_block *nb, spin_unlock(&synthetic_tsc_lock); break; #ifdef CONFIG_HOTPLUG_CPU - case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: - case CPU_DEAD: - case CPU_DEAD_FROZEN: + case CPU_DOWN_PREPARE: + case CPU_DOWN_PREPARE_FROZEN: spin_lock(&synthetic_tsc_lock); if (synthetic_tsc_refcount) disable_synthetic_tsc(hotcpu); |