/* * Cell Broadband Engine OProfile Support * * (C) Copyright IBM Corporation 2006 * * Author: David Erb (djerb@us.ibm.com) * Modifications: * Carl Love * Maynard Johnson * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../platforms/cell/interrupt.h" #include "cell/pr_util.h" static void cell_global_stop_spu(void); /* * spu_cycle_reset is the number of cycles between samples. * This variable is used for SPU profiling and should ONLY be set * at the beginning of cell_reg_setup; otherwise, it's read-only. */ static unsigned int spu_cycle_reset; #define NUM_SPUS_PER_NODE 8 #define SPU_CYCLES_EVENT_NUM 2 /* event number for SPU_CYCLES */ #define PPU_CYCLES_EVENT_NUM 1 /* event number for CYCLES */ #define PPU_CYCLES_GRP_NUM 1 /* special group number for identifying * PPU_CYCLES event */ #define CBE_COUNT_ALL_CYCLES 0x42800000 /* PPU cycle event specifier */ #define NUM_THREADS 2 /* number of physical threads in * physical processor */ #define NUM_DEBUG_BUS_WORDS 4 #define NUM_INPUT_BUS_WORDS 2 #define MAX_SPU_COUNT 0xFFFFFF /* maximum 24 bit LFSR value */ struct pmc_cntrl_data { unsigned long vcntr; unsigned long evnts; unsigned long masks; unsigned long enabled; }; /* * ibm,cbe-perftools rtas parameters */ struct pm_signal { u16 cpu; /* Processor to modify */ u16 sub_unit; /* hw subunit this applies to (if applicable)*/ short int signal_group; /* Signal Group to Enable/Disable */ u8 bus_word; /* Enable/Disable on this Trace/Trigger/Event * Bus Word(s) (bitmask) */ u8 bit; /* Trigger/Event bit (if applicable) */ }; /* * rtas call arguments */ enum { SUBFUNC_RESET = 1, SUBFUNC_ACTIVATE = 2, SUBFUNC_DEACTIVATE = 3, PASSTHRU_IGNORE = 0, PASSTHRU_ENABLE = 1, PASSTHRU_DISABLE = 2, }; struct pm_cntrl { u16 enable; u16 stop_at_max; u16 trace_mode; u16 freeze; u16 count_mode; }; static struct { u32 group_control; u32 debug_bus_control; struct pm_cntrl pm_cntrl; u32 pm07_cntrl[NR_PHYS_CTRS]; } pm_regs; #define GET_SUB_UNIT(x) ((x & 0x0000f000) >> 12) #define GET_BUS_WORD(x) ((x & 0x000000f0) >> 4) #define GET_BUS_TYPE(x) ((x & 0x00000300) >> 8) #define GET_POLARITY(x) ((x & 0x00000002) >> 1) #define GET_COUNT_CYCLES(x) (x & 0x00000001) #define GET_INPUT_CONTROL(x) ((x & 0x00000004) >> 2) static DEFINE_PER_CPU(unsigned long[NR_PHYS_CTRS], pmc_values); static struct pmc_cntrl_data pmc_cntrl[NUM_THREADS][NR_PHYS_CTRS]; /* * The CELL profiling code makes rtas calls to setup the debug bus to * route the performance signals. Additionally, SPU profiling requires * a second rtas call to setup the hardware to capture the SPU PCs. * The EIO error value is returned if the token lookups or the rtas * call fail. The EIO error number is the best choice of the existing * error numbers. The probability of rtas related error is very low. But * by returning EIO and printing additional information to dmsg the user * will know that OProfile did not start and dmesg will tell them why. * OProfile does not support returning errors on Stop. Not a huge issue * since failure to reset the debug bus or stop the SPU PC collection is * not a fatel issue. Chances are if the Stop failed, Start doesn't work * either. */ /* * Interpetation of hdw_thread: * 0 - even virtual cpus 0, 2, 4,... * 1 - odd virtual cpus 1, 3, 5, ... * * FIXME: this is strictly wrong, we need to clean this up in a number * of places. It works for now. -arnd */ static u32 hdw_thread; static u32 virt_cntr_inter_mask; static struct timer_list timer_virt_cntr; /* * pm_signal needs to be global since it is initialized in * cell_reg_setup at the time when the necessary information * is available. */ static struct pm_signal pm_signal[NR_PHYS_CTRS]; static int pm_rtas_token; /* token for debug bus setup call */ static int spu_rtas_token; /* token for SPU cycle profiling */ static u32 reset_value[NR_PHYS_CTRS]; static int num_counters; static int oprofile_running; static DEFINE_SPINLOCK(virt_cntr_lock); static u32 ctr_enabled; static unsigned char input_bus[NUM_INPUT_BUS_WORDS]; /* * Firmware interface functions */ static int rtas_ibm_cbe_perftools(int subfunc, int passthru, void *address, unsigned long length) { u64 paddr = __pa(address); return rtas_call(pm_rtas_token, 5, 1, NULL, subfunc, passthru, paddr >> 32, paddr & 0xffffffff, length); } static void pm_rtas_reset_signals(u32 node) { int ret; struct pm_signal pm_signal_local; /* * The debug bus is being set to the passthru disable state. * However, the FW still expects atleast one legal signal routing * entry or it will return an error on the arguments. If we don't * supply a valid entry, we must ignore all return values. Ignoring * all return values means we might miss an error we should be * concerned about. */ /* fw expects physical cpu #. */ pm_signal_local.cpu = node; pm_signal_local.signal_group = 21; pm_signal_local.bus_word = 1; pm_signal_local.sub_unit = 0; pm_signal_local.bit = 0; ret = rtas_ibm_cbe_perftools(SUBFUNC_RESET, PASSTHRU_DISABLE, &pm_signal_local, sizeof(struct pm_signal)); if (unlikely(ret)) /* * Not a fatal error. For Oprofile stop, the oprofile * functions do not support returning an error for * failure to stop OProfile. */ printk(KERN_WARNING "%s: rtas returned: %d\n", __func__, ret); } static int pm_rtas_activate_signals(u32 node, u32 count) { int ret; int i, j; struct pm_signal pm_signal_local[NR_PHYS_CTRS]; /* * There is no debug setup required for the cycles event. * Note that only events in the same group can be used. * Otherwise, there will be conflicts in correctly routing * the signals on the debug bus. It is the responsiblity * of the OProfile user tool to check the events are in * the same group. */ i = 0; for (j = 0; j < count; j++) { if (pm_signal[j].signal_group != PPU_CYCLES_GRP_NUM) { /* fw expects physical cpu # */ pm_signal_local[i].cpu = node; pm_signal_local[i].signal_group = pm_signal[j].signal_group; pm_signal_local[i].bus_word = pm_signal[j].bus_word; pm_signal_local[i].sub_unit = pm_signal[j].sub_unit; pm_signal_local[i].bit = pm_signal[j].bit; i++; } } if (i != 0) { ret = rtas_ibm_cbe_perftools(SUBFUNC_ACTIVATE, PASSTHRU_ENABLE, pm_signal_local, i * sizeof(struct pm_signal)); if (unlikely(ret)) { printk(KERN_WARNING "%s: rtas returned: %d\n", __func__, ret); return -EIO; } } return 0; } /* * PM Signal functions */ static void set_pm_event(u32 ctr, int event, u32 unit_mask) { struct pm_signal *p; u32 signal_bit; u32 bus_word, bus_type, count_cycles, polarity, input_control; int j, i; if (event == PPU_CYCLES_EVENT_NUM) { /* Special Event: Count all cpu cycles */ pm_regs.pm07_cntrl[ctr] = CBE_COUNT_ALL_CYCLES; p = &(pm_signal[ctr]); p->signal_group = PPU_CYCLES_GRP_NUM; p->bus_word = 1; p->sub_unit = 0; p->bit = 0; goto out; } else { pm_regs.pm07_cntrl[ctr] = 0; } bus_word = GET_BUS_WORD(unit_mask); bus_type = GET_BUS_TYPE(unit_mask); count_cycles = GET_COUNT_CYCLES(unit_mask); polarity = GET_POLARITY(unit_mask); input_control = GET_INPUT_CONTROL(unit_mask); signal_bit = (event % 100); p = &(pm_signal[ctr]); p->signal_group = event / 100; p->bus_word = bus_word; p->sub_unit = GET_SUB_UNIT(unit_mask); pm_regs.pm07_cntrl[ctr] = 0; pm_regs.pm07_cntrl[ctr] |= PM07_CTR_COUNT_CYCLES(count_cycles); pm_regs.pm07_cntrl[ctr] |= PM07_CTR_POLARITY(polarity); pm_regs.pm07_cntrl[ctr] |= PM07_CTR_INPUT_CONTROL(input_control); /* * Some of the islands signal selection is based on 64 bit words. * The debug bus words are 32 bits, the input words to the performance * counters are defined as 32 bits. Need to convert the 64 bit island * specification to the appropriate 32 input bit and bus word for the * performance counter event selection. See the CELL Performance * monitoring signals manual and the Perf cntr hardware descriptions * for the details. */ if (input_control == 0) { if (signal_bit > 31) { signal_bit -= 32; if (bus_word == 0x3) bus_word = 0x2; else if (bus_word == 0xc) bus_word = 0x8; } if ((bus_type == 0) && p->signal_group >= 60) bus_type = 2; if ((bus_type == 1) && p->signal_group >= 50) bus_type = 0; pm_regs.pm07_cntrl[ctr] |= PM07_CTR_INPUT_MUX(signal_bit); } else { pm_regs.pm07_cntrl[ctr] = 0; p->bit = signal_bit; } for (i = 0; i < NUM_DEBUG_BUS_WORDS; i++) { if (bus_word & (1 << i)) { pm_regs.debug_bus_control |= (bus_type << (30 - (2 * i))); for (j = 0; j < NUM_INPUT_BUS_WORDS; j++) { if (input_bus[j] == 0xff) { input_bus[j] = i; pm_regs.group_control |= (i << (30 - (2 * j))); break; } } } } out: ; } static void write_pm_cntrl(int cpu) { /* * Oprofile will use 32 bit counters, set bits 7:10 to 0 * pmregs.pm_cntrl is a global */ u32 val = 0; if (pm_regs.pm_cntrl.enable == 1) val |= CBE_PM_ENABLE_PERF_MON; if (pm_regs.pm_cntrl.stop_at_max == 1) val |= CBE_PM_STOP_AT_MAX; if (pm_regs.pm_cntrl.trace_mode == 1) val |= CBE_PM_TRACE_MODE_SET(pm_regs.pm_cntrl.trace_mode); if (pm_regs.pm_cntrl.freeze == 1) val |= CBE_PM_FREEZE_ALL_CTRS; /* * Routine set_count_mode must be called previously to set * the count mode based on the user selection of user and kernel. */ val |= CBE_PM_COUNT_MODE_SET(pm_regs.pm_cntrl.count_mode); cbe_write_pm(cpu, pm_control, val); } static inline void set_count_mode(u32 kernel, u32 user) { /* * The user must specify user and kernel if they want them. If * neither is specified, OProfile will count in hypervisor mode. * pm_regs.pm_cntrl is a global */ if (kernel) { if (user) pm_regs.pm_cntrl.count_mode = CBE_COUNT_ALL_MODES; else pm_regs.pm_cntrl.count_mode = CBE_COUNT_SUPERVISOR_MODE; } else { if (user) pm_regs.pm_cntrl.count_mode = CBE_COUNT_PROBLEM_MODE; else pm_regs.pm_cntrl.count_mode = CBE_COUNT_HYPERVISOR_MODE; } } static inline void enable_ctr(u32 cpu, u32 ctr, u32 *pm07_cntrl) { pm07_cntrl[ctr] |= CBE_PM_CTR_ENABLE; cbe_write_pm07_control(cpu, ctr, pm07_cntrl[ctr]); } /* * Oprofile is expected to collect data on all CPUs simultaneously. * However, there is one set of performance counters per node. There are * two hardware threads or virtual CPUs on each node. Hence, OProfile must * multiplex in time the performance counter collection on the two virtual * CPUs. The multiplexing of the performance counters is done by this * virtual counter routine. * * The pmc_values used below is defined as 'per-cpu' but its use is * more akin to 'per-node'. We need to store two sets of counter * values per node -- one for the previous run and one for the next. * The per-cpu[NR_PHYS_CTRS] gives us the storage we need. Each odd/even * pair of per-cpu arrays is used for storing the previous and next * pmc values for a given node. * NOTE: We use the per-cpu variable to improve cache performance. * * This routine will alternate loading the virtual counters for * virtual CPUs */ static void cell_virtual_cntr(unsigned long data) { int i, prev_hdw_thread, next_hdw_thread; u32 cpu; unsigned long flags; /* * Make sure that the interrupt_hander and the virt counter are * not both playing with the counters on the same node. */ spin_lock_irqsave(&virt_cntr_lock, flags); prev_hdw_thread = hdw_thread; /* switch the cpu handling the interrupts */ hdw_thread = 1 ^ hdw_thread; next_hdw_thread = hdw_thread; pm_regs.group_control = 0; pm_regs.debug_bus_control = 0; for (i = 0; i < NUM_INPUT_BUS_WORDS; i++) input_bus[i] = 0xff; /* * There are some per thread events. Must do the * set event, for the thread that is being started */ for (i = 0; i < num_counters; i++) set_pm_event(i, pmc_cntrl[next_hdw_thread][i].evnts, pmc_cntrl[next_hdw_thread][i].masks); /* * The following is done only once per each node, but * we need cpu #, not node #, to pass to the cbe_xxx functions. */ for_each_online_cpu(cpu) { if (cbe_get_hw_thread_id(cpu)) continue; /* * stop counters, save counter values, restore counts * for previous thread */ cbe_disable_pm(cpu); cbe_disable_pm_interrupts(cpu); for (i = 0; i < num_counters; i++) { per_cpu(pmc_values, cpu + prev_hdw_thread)[i] = cbe_read_ctr(cpu, i); if (per_cpu(pmc_values, cpu + next_hdw_thread)[i] == 0xFFFFFFFF) /* If the cntr value is 0xffffffff, we must * reset that to 0xfffffff0 when the current * thread is restarted. This will generate a * new interrupt and make sure that we never * restore the counters to the max value. If * the counters were restored to the max value, * they do not increment and no interrupts are * generated. Hence no more samples will be * collected on that cpu. */ cbe_write_ctr(cpu, i, 0xFFFFFFF0); else cbe_write_ctr(cpu, i, per_cpu(pmc_values, cpu + next_hdw_thread)[i]); } /* * Switch to the other thread. Change the interrupt * and control regs to be scheduled on the CPU * corresponding to the thread to execute. */ for (i = 0; i < num_counters; i++) { if (pmc_cntrl[next_hdw_thread][i].enabled) { /* * There are some per thread events. * Must do the set event, enable_cntr * for each cpu. */ enable_ctr(cpu, i, pm_regs.pm07_cntrl); } else { cbe_write_pm07_control(cpu, i, 0); } } /* Enable interrupts on the CPU thread that is starting */ cbe_enable_pm_interrupts(cpu, next_hdw_thread, virt_cntr_inter_mask); cbe_enable_pm(cpu); } spin_unlock_irqrestore(&virt_cntr_lock, flags); mod_timer(&timer_virt_cntr, jiffies + HZ / 10); } static void start_virt_cntrs(void) { init_timer(&timer_virt_cntr); timer_virt_cntr.function = cell_virtual_cntr; timer_virt_cntr.data = 0UL; timer_virt_cntr.expires = jiffies + HZ / 10; add_timer(&timer_virt_cntr); } /* This function is called once for all cpus combined */ static int cell_reg_setup(struct op_counter_config *ctr, struct op_system_config *sys, int num_ctrs) { int i, j, cpu; spu_cycle_reset = 0; if (ctr[0].event == SPU_CYCLES_EVENT_NUM) { spu_cycle_reset = ctr[0].count; /* * Each node will need to make the rtas call to start * and stop SPU profiling. Get the token once and store it. */ spu_rtas_token = rtas_token("ibm,cbe-spu-perftools"); if (unlikely(spu_rtas_token == RTAS_UNKNOWN_SERVICE)) { printk(KERN_ERR "%s: rtas token ibm,cbe-spu-perftools unknown\n", __func__); return -EIO; } } pm_rtas_token = rtas_token("ibm,cbe-perftools"); /* * For all events excetp PPU CYCLEs, each node will need to make * the rtas cbe-perftools call to setup and reset the debug bus. * Make the token lookup call once and store it in the global * variable pm_rtas_token. */ if (unlikely(pm_rtas_token == RTAS_UNKNOWN_SERVICE)) { printk(KERN_ERR "%s: rtas token ibm,cbe-perftools unknown\n", __func__); return -EIO; } num_counters = num_ctrs; pm_regs.group_control = 0; pm_regs.debug_bus_control = 0; /* setup the pm_control register */ memset(&pm_regs.pm_cntrl, 0, sizeof(struct pm_cntrl)); pm_regs.pm_cntrl.stop_at_max = 1; pm_regs.pm_cntrl.trace_mode = 0; pm_regs.pm_cntrl.freeze = 1; set_count_mode(sys->enable_kernel, sys->enable_user); /* Setup the thread 0 events */ for (i = 0; i < num_ctrs; ++i) { pmc_cntrl[0][i].evnts = ctr[i].event; pmc_cntrl[0][i].masks = ctr[i].unit_mask; pmc_cntrl[0][i].enabled = ctr[i].enabled; pmc_cntrl[0][i].vcntr = i; for_each_possible_cpu(j) per_cpu(pmc_values, j)[i] = 0; } /* * Setup the thread 1 events, map the thread 0 event to the * equivalent thread 1 event. */ for (i = 0; i < num_ctrs; ++i) { if ((ctr[i].event >= 2100) && (ctr[i].event <= 2111)) pmc_cntrl[1][i].evnts = ctr[i].event + 19; else if (ctr[i].event == 2203) pmc_cntrl[1][i].evnts = ctr[i].event; else if ((ctr[i].event >= 2200) && (ctr[i].event <= 2215)) pmc_cntrl[1][i].evnts = ctr[i].event + 16; else pmc_cntrl[1][i].evnts = ctr[i].event; pmc_cntrl[1][i].masks = ctr[i].unit_mask; pmc_cntrl[1][i].enabled = ctr[i].enabled; pmc_cntrl[1][i].vcntr = i; } for (i = 0; i < NUM_INPUT_BUS_WORDS; i++) input_bus[i] = 0xff; /* * Our counters count up, and "count" refers to * how much before the next interrupt, and we interrupt * on overflow. So we calculate the starting value * which will give us "count" until overflow. * Then we set the events on the enabled counters. */ for (i = 0; i < num_counters; ++i) { /* start with virtual counter set 0 */ if (pmc_cntrl[0][i].enabled) { /* Using 32bit counters, reset max - count */ reset_value[i] = 0xFFFFFFFF - ctr[i].count; set_pm_event(i, pmc_cntrl[0][i].evnts, pmc_cntrl[0][i].masks); /* global, used by cell_cpu_setup */ ctr_enabled |= (1 << i); } } /* initialize the previous counts for the virtual cntrs */ for_each_online_cpu(cpu) for (i = 0; i < num_counters; ++i) { per_cpu(pmc_values, cpu)[i] = reset_value[i]; } return 0; } /* This function is called once for each cpu */ static int cell_cpu_setup(struct op_counter_config *cntr) { u32 cpu = smp_processor_id(); u32 num_enabled = 0; int i; if (spu_cycle_reset) return 0; /* There is one performance monitor per processor chip (i.e. node), * so we only need to perform this function once per node. */ if (cbe_get_hw_thread_id(cpu)) return 0; /* Stop all counters */ cbe_disable_pm(cpu); cbe_disable_pm_interrupts(cpu); cbe_write_pm(cpu, pm_interval, 0); cbe_write_pm(cpu, pm_start_stop, 0); cbe_write_pm(cpu, group_control, pm_regs.group_control); cbe_write_pm(cpu, debug_bus_control, pm_regs.debug_bus_control); write_pm_cntrl(cpu); for (i = 0; i < num_counters; ++i) { if (ctr_enabled & (1 << i)) { pm_signal[num_enabled].cpu = cbe_cpu_to_node(cpu); num_enabled++; } } /* * The pm_rtas_activate_signals will return -EIO if the FW * call failed. */ return pm_rtas_activate_signals(cbe_cpu_to_node(cpu), num_enabled); } #define ENTRIES 303 #define MAXLFSR 0xFFFFFF /* precomputed table of 24 bit LFSR values */ static int initial_lfsr[] = { 8221349, 12579195, 5379618, 10097839, 7512963, 7519310, 3955098, 10753424, 15507573, 7458917, 285419, 2641121, 9780088, 3915503, 6668768, 1548716, 4885000, 8774424, 9650099, 2044357, 2304411, 9326253, 10332526, 4421547, 3440748, 10179459, 13332843, 10375561, 1313462, 8375100, 5198480, 6071392, 9341783, 1526887, 3985002, 1439429, 13923762, 7010104, 11969769, 4547026, 2040072, 4025602, 3437678, 7939992, 11444177, 4496094, 9803157, 10745556, 3671780, 4257846, 5662259, 13196905, 3237343, 12077182, 16222879, 7587769, 14706824, 2184640, 12591135, 10420257, 7406075, 3648978, 11042541, 15906893, 11914928, 4732944, 10695697, 12928164, 11980531, 4430912, 11939291, 2917017, 6119256, 4172004, 9373765, 8410071, 14788383, 5047459, 5474428, 1737756, 15967514, 13351758, 6691285, 8034329, 2856544, 14394753, 11310160, 12149558, 7487528, 7542781, 15668898, 12525138, 12790975, 3707933, 9106617, 1965401, 16219109, 12801644, 2443203, 4909502, 8762329, 3120803, 6360315, 9309720, 15164599, 10844842, 4456529, 6667610, 14924259, 884312, 6234963, 3326042, 15973422, 13919464, 5272099, 6414643, 3909029, 2764324, 5237926, 4774955, 10445906, 4955302, 5203726, 10798229, 11443419, 2303395, 333836, 9646934, 3464726, 4159182, 568492, 995747, 10318756, 13299332, 4836017, 8237783, 3878992, 2581665, 11394667, 5672745, 14412947, 3159169, 9094251, 16467278, 8671392, 15230076, 4843545, 7009238, 15504095, 1494895, 9627886, 14485051, 8304291, 252817, 12421642, 16085736, 4774072, 2456177, 4160695, 15409741, 4902868, 5793091, 13162925, 16039714, 782255, 11347835, 14884586, 366972, 16308990, 11913488, 13390465, 2958444, 10340278, 1177858, 1319431, 10426302, 2868597, 126119, 5784857, 5245324, 10903900, 16436004, 3389013, 1742384, 14674502, 10279218, 8536112, 10364279, 6877778, 14051163, 1025130, 6072469, 1988305, 8354440, 8216060, 16342977, 13112639, 3976679, 5913576, 8816697, 6879995, 14043764, 3339515, 9364420, 15808858, 12261651, 2141560, 5636398, 10345425, 10414756, 781725, 6155650, 4746914, 5078683, 7469001, 6799140, 10156444, 9667150, 10116470, 4133858, 2121972, 1124204, 1003577, 1611214, 14304602, 16221850, 13878465, 13577744, 3629235, 8772583, 10881308, 2410386, 7300044, 5378855, 9301235, 12755149, 4977682, 8083074, 10327581, 6395087, 9155434, 15501696, 7514362, 14520507, 15808945, 3244584, 4741962, 9658130, 14336147, 8654727, 7969093, 15759799, 14029445, 5038459, 9894848, 8659300, 13699287, 8834306, 10712885, 14753895, 10410465, 3373251, 309501, 9561475, 5526688, 14647426, 14209836, 5339224, 207299, 14069911, 8722990, 2290950, 3258216, 12505185, 6007317, 9218111, 14661019, 10537428, 11731949, 9027003, 6641507, 9490160, 200241, 9720425, 16277895, 10816638, 1554761, 10431375, 7467528, 6790302, 3429078, 14633753, 14428997, 11463204, 3576212, 2003426, 6123687, 820520, 9992513, 15784513, 5778891, 6428165, 8388607 }; /* * The hardware uses an LFSR counting sequence to determine when to capture * the SPU PCs. An LFSR sequence is like a puesdo random number sequence * where each number occurs once in the sequence but the sequence is not in * numerical order. The SPU PC capture is done when the LFSR sequence reaches * the last value in the sequence. Hence the user specified value N * corresponds to the LFSR number that is N from the end of the sequence. * * To avoid the time to compute the LFSR, a lookup table is used. The 24 bit * LFSR sequence is broken into four ranges. The spacing of the precomputed * values is adjusted in each range so the error between the user specifed * number (N) of events between samples and the actual number of events based * on the precomputed value will be les then about 6.2%. Note, if the user * specifies N < 2^16, the LFSR value that is 2^16 from the end will be used. * This is to prevent the loss of samples because the trace buffer is full. * * User specified N Step between Index in * precomputed values precomputed * table * 0 to 2^16-1 ---- 0 * 2^16 to 2^16+2^19-1 2^12 1 to 128 * 2^16+2^19 to 2^16+2^19+2^22-1 2^15 129 to 256 * 2^16+2^19+2^22 to 2^24-1 2^18 257 to 302 * * * For example, the LFSR values in the second range are computed for 2^16, * 2^16+2^12, ... , 2^19-2^16, 2^19 and stored in the table at indicies * 1, 2,..., 127, 128. * * The 24 bit LFSR value for the nth number in the sequence can be * calculated using the following code: * * #define size 24 * int calculate_lfsr(int n) * { * int i; * unsigned int newlfsr0; * unsigned int lfsr = 0xFFFFFF; * unsigned int howmany = n; * * for (i = 2; i < howmany + 2; i++) { * newlfsr0 = (((lfsr >> (size - 1 - 0)) & 1) ^ * ((lfsr >> (size - 1 - 1)) & 1) ^ * (((lfsr >> (size - 1 - 6)) & 1) ^ * ((lfsr >> (size - 1 - 23)) & 1))); * * lfsr >>= 1; * lfsr = lfsr | (newlfsr0 << (size - 1)); * } * return lfsr; * } */ #define V2_16 (0x1 << 16) #define V2_19 (0x1 << 19) #define V2_22 (0x1 << 22) static int calculate_lfsr(int n) { /* * The ranges and steps are in powers of 2 so the calculations * can be done using shifts rather then divide. */ int index; if ((n >> 16) == 0) index = 0; else if (((n - V2_16) >> 19) == 0) index = ((n - V2_16) >> 12) + 1; else if (((n - V2_16 - V2_19) >> 22) == 0) index = ((n - V2_16 - V2_19) >> 15 ) + 1 + 128; else if (((n - V2_16 - V2_19 - V2_22) >> 24) == 0) index = ((n - V2_16 - V2_19 - V2_22) >> 18 ) + 1 + 256; else index = ENTRIES-1; /* make sure index is valid */ if ((index > ENTRIES) || (index < 0)) index = ENTRIES-1; return initial_lfsr[index]; } static int pm_rtas_activate_spu_profiling(u32 node) { int ret, i; struct pm_signal pm_signal_local[NR_PHYS_CTRS]; /* * Set up the rtas call to configure the debug bus to * route the SPU PCs. Setup the pm_signal for each SPU */ for (i = 0; i < NUM_SPUS_PER_NODE; i++) { pm_signal_local[i].cpu = node; pm_signal_local[i].signal_group = 41; /* spu i on word (i/2) */ pm_signal_local[i].bus_word = 1 << i / 2; /* spu i */ pm_signal_local[i].sub_unit = i; pm_signal_local[i].bit = 63; } ret = rtas_ibm_cbe_perftools(SUBFUNC_ACTIVATE, PASSTHRU_ENABLE, pm_signal_local, (NUM_SPUS_PER_NODE * sizeof(struct pm_signal))); if (unlikely(ret)) { printk(KERN_WARNING "%s: rtas returned: %d\n", __func__, ret); return -EIO; } return 0; } #ifdef CONFIG_CPU_FREQ static int oprof_cpufreq_notify(struct notifier_block *nb, unsigned long val, void *data) { int ret = 0; struct cpufreq_freqs *frq = data; if ((val == CPUFREQ_PRECHANGE && frq->old < frq->new) || (val == CPUFREQ_POSTCHANGE && frq->old > frq->new) || (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) set_spu_profiling_frequency(frq->new, spu_cycle_reset); return ret; } static struct notifier_block cpu_freq_notifier_block = { .notifier_call = oprof_cpufreq_notify }; #endif static int cell_global_start_spu(struct op_counter_config *ctr) { int subfunc; unsigned int lfsr_value; int cpu; int ret; int rtas_error; unsigned int cpu_khzfreq = 0; /* The SPU profiling uses time-based profiling based on * cpu frequency, so if configured with the CPU_FREQ * option, we should detect frequency changes and react * accordingly. */ #ifdef CONFIG_CPU_FREQ ret = cpufreq_register_notifier(&cpu_freq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER); if (ret < 0) /* this is not a fatal error */ printk(KERN_ERR "CPU freq change registration failed: %d\n", ret); else cpu_khzfreq = cpufreq_quick_get(smp_processor_id()); #endif set_spu_profiling_frequency(cpu_khzfreq, spu_cycle_reset); for_each_online_cpu(cpu) { if (cbe_get_hw_thread_id(cpu)) continue; /* * Setup SPU cycle-based profiling. * Set perf_mon_control bit 0 to a zero before * enabling spu collection hardware. */ cbe_write_pm(cpu, pm_control, 0); if (spu_cycle_reset > MAX_SPU_COUNT) /* use largest possible value */ lfsr_value = calculate_lfsr(MAX_SPU_COUNT-1); else lfsr_value = calculate_lfsr(spu_cycle_reset); /* must use a non zero value. Zero disables data collection. */ if (lfsr_value == 0) lfsr_value = calculate_lfsr(1); lfsr_value = lfsr_value << 8; /* shift lfsr to correct * register location */ /* debug bus setup */ ret = pm_rtas_activate_spu_profiling(cbe_cpu_to_node(cpu)); if (unlikely(ret)) { rtas_error = ret; goto out; } subfunc = 2; /* 2 - activate SPU tracing, 3 - deactivate */ /* start profiling */ ret = rtas_call(spu_rtas_token, 3, 1, NULL, subfunc, cbe_cpu_to_node(cpu), lfsr_value); if (unlikely(ret != 0)) { printk(KERN_ERR "%s: rtas call ibm,cbe-spu-perftools failed, return = %d\n", __func__, ret); rtas_error = -EIO; goto out; } } rtas_error = start_spu_profiling(spu_cycle_reset); if (rtas_error) goto out_stop; oprofile_running = 1; return 0; out_stop: cell_global_stop_spu(); /* clean up the PMU/debug bus */ out: return rtas_error; } static int cell_global_start_ppu(struct op_counter_config *ctr) { u32 cpu, i; u32 interrupt_mask = 0; /* This routine gets called once for the system. * There is one performance monitor per node, so we * only need to perform this function once per node. */ for_each_online_cpu(cpu) { if (cbe_get_hw_thread_id(cpu)) continue; interrupt_mask = 0; for (i = 0; i < num_counters; ++i) { if (ctr_enabled & (1 << i)) { cbe_write_ctr(cpu, i, reset_value[i]); enable_ctr(cpu, i, pm_regs.pm07_cntrl); interrupt_mask |= CBE_PM_CTR_OVERFLOW_INTR(i); } else { /* Disable counter */ cbe_write_pm07_control(cpu, i, 0); } } cbe_get_and_clear_pm_interrupts(cpu); cbe_enable_pm_interrupts(cpu, hdw_thread, interrupt_mask); cbe_enable_pm(cpu); } virt_cntr_inter_mask = interrupt_mask; oprofile_running = 1; smp_wmb(); /* * NOTE: start_virt_cntrs will result in cell_virtual_cntr() being * executed which manipulates the PMU. We start the "virtual counter" * here so that we do not need to synchronize access to the PMU in * the above for-loop. */ start_virt_cntrs(); return 0; } static int cell_global_start(struct op_counter_config *ctr) { if (spu_cycle_reset) return cell_global_start_spu(ctr); else return cell_global_start_ppu(ctr); } /* * Note the generic OProfile stop calls do not support returning * an error on stop. Hence, will not return an error if the FW * calls fail on stop. Failure to reset the debug bus is not an issue. * Failure to disable the SPU profiling is not an issue. The FW calls * to enable the performance counters and debug bus will work even if * the hardware was not cleanly reset. */ static void cell_global_stop_spu(void) { int subfunc, rtn_value; unsigned int lfsr_value; int cpu; oprofile_running = 0; #ifdef CONFIG_CPU_FREQ cpufreq_unregister_notifier(&cpu_freq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER); #endif for_each_online_cpu(cpu) { if (cbe_get_hw_thread_id(cpu)) continue; subfunc = 3; /* * 2 - activate SPU tracing, * 3 - deactivate */ lfsr_value = 0x8f100000; rtn_value = rtas_call(spu_rtas_token, 3, 1, NULL, subfunc, cbe_cpu_to_node(cpu), lfsr_value); if (unlikely(rtn_value != 0)) { printk(KERN_ERR "%s: rtas call ibm,cbe-spu-perftools failed, return = %d\n", __func__, rtn_value); } /* Deactivate the signals */ pm_rtas_reset_signals(cbe_cpu_to_node(cpu)); } stop_spu_profiling(); } static void cell_global_stop_ppu(void) { int cpu; /* * This routine will be called once for the system. * There is one performance monitor per node, so we * only need to perform this function once per node. */ del_timer_sync(&timer_virt_cntr); oprofile_running = 0; smp_wmb(); for_each_online_cpu(cpu) { if (cbe_get_hw_thread_id(cpu)) continue; cbe_sync_irq(cbe_cpu_to_node(cpu)); /* Stop the counters */ cbe_disable_pm(cpu); /* Deactivate the signals */ pm_rtas_reset_signals(cbe_cpu_to_node(cpu)); /* Deactivate interrupts */ cbe_disable_pm_interrupts(cpu); } } static void cell_global_stop(void) { if (spu_cycle_reset) cell_global_stop_spu(); else cell_global_stop_ppu(); } static void cell_handle_interrupt(struct pt_regs *regs, struct op_counter_config *ctr) { u32 cpu; u64 pc; int is_kernel; unsigned long flags = 0; u32 interrupt_mask; int i; cpu = smp_processor_id(); /* * Need to make sure the interrupt handler and the virt counter * routine are not running at the same time. See the * cell_virtual_cntr() routine for additional comments. */ spin_lock_irqsave(&virt_cntr_lock, flags); /* * Need to disable and reenable the performance counters * to get the desired behavior from the hardware. This * is hardware specific. */ cbe_disable_pm(cpu); interrupt_mask = cbe_get_and_clear_pm_interrupts(cpu); /* * If the interrupt mask has been cleared, then the virt cntr * has cleared the interrupt. When the thread that generated * the interrupt is restored, the data count will be restored to * 0xffffff0 to cause the interrupt to be regenerated. */ if ((oprofile_running == 1) && (interrupt_mask != 0)) { pc = regs->nip; is_kernel = is_kernel_addr(pc); for (i = 0; i < num_counters; ++i) { if ((interrupt_mask & CBE_PM_CTR_OVERFLOW_INTR(i)) && ctr[i].enabled) { oprofile_add_ext_sample(pc, regs, i, is_kernel); cbe_write_ctr(cpu, i, reset_value[i]); } } /* * The counters were frozen by the interrupt. * Reenable the interrupt and restart the counters. * If there was a race between the interrupt handler and * the virtual counter routine. The virutal counter * routine may have cleared the interrupts. Hence must * use the virt_cntr_inter_mask to re-enable the interrupts. */ cbe_enable_pm_interrupts(cpu, hdw_thread, virt_cntr_inter_mask); /* * The writes to the various performance counters only writes * to a latch. The new values (interrupt setting bits, reset * counter value etc.) are not copied to the actual registers * until the performance monitor is enabled. In order to get * this to work as desired, the permormance monitor needs to * be disabled while writing to the latches. This is a * HW design issue. */ cbe_enable_pm(cpu); } spin_unlock_irqrestore(&virt_cntr_lock, flags); } /* * This function is called from the generic OProfile * driver. When profiling PPUs, we need to do the * generic sync start; otherwise, do spu_sync_start. */ static int cell_sync_start(void) { if (spu_cycle_reset) return spu_sync_start(); else return DO_GENERIC_SYNC; } static int cell_sync_stop(void) { if (spu_cycle_reset) return spu_sync_stop(); else return 1; } struct op_powerpc_model op_model_cell = { .reg_setup = cell_reg_setup, .cpu_setup = cell_cpu_setup, .global_start = cell_global_start, .global_stop = cell_global_stop, .sync_start = cell_sync_start, .sync_stop = cell_sync_stop, .handle_interrupt = cell_handle_interrupt, };