diff options
Diffstat (limited to 'drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_pm_driver.c')
| -rwxr-xr-x | drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_pm_driver.c | 911 |
1 files changed, 911 insertions, 0 deletions
diff --git a/drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_pm_driver.c b/drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_pm_driver.c new file mode 100755 index 00000000000..1362b98ac6e --- /dev/null +++ b/drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_pm_driver.c @@ -0,0 +1,911 @@ +/* + * + * (C) COPYRIGHT 2010-2013 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_pm_driver.c + * Base kernel Power Management hardware control + */ + +#include <kbase/src/common/mali_kbase.h> +#include <kbase/src/common/mali_midg_regmap.h> +#include <kbase/src/common/mali_kbase_gator.h> +#include <kbase/src/common/mali_kbase_pm.h> + +#if MALI_MOCK_TEST +#define MOCKABLE(function) function##_original +#else +#define MOCKABLE(function) function +#endif /* MALI_MOCK_TEST */ + +/** Actions that can be performed on a core. + * + * This enumeration is private to the file. Its values are set to allow @ref core_type_to_reg function, + * which decodes this enumeration, to be simpler and more efficient. + */ +typedef enum kbasep_pm_action { + ACTION_PRESENT = 0, + ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO), + ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO), + ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO), + ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO), + ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO) +} kbasep_pm_action; + +/** Decode a core type and action to a register. + * + * Given a core type (defined by @ref kbase_pm_core_type) and an action (defined by @ref kbasep_pm_action) this + * function will return the register offset that will perform the action on the core type. The register returned is + * the \c _LO register and an offset must be applied to use the \c _HI register. + * + * @param core_type The type of core + * @param action The type of action + * + * @return The register offset of the \c _LO register that performs an action of type \c action on a core of type \c + * core_type. + */ +static u32 core_type_to_reg(kbase_pm_core_type core_type, kbasep_pm_action action) +{ + return core_type + action; +} + +/** Invokes an action on a core set + * + * This function performs the action given by \c action on a set of cores of a type given by \c core_type. It is a + * static function used by @ref kbase_pm_transition_core_type + * + * @param kbdev The kbase device structure of the device + * @param core_type The type of core that the action should be performed on + * @param cores A bit mask of cores to perform the action on (low 32 bits) + * @param action The action to perform on the cores + */ +STATIC void kbase_pm_invoke(kbase_device *kbdev, kbase_pm_core_type core_type, u64 cores, kbasep_pm_action action) +{ + u32 reg; + u32 lo = cores & 0xFFFFFFFF; + u32 hi = (cores >> 32) & 0xFFFFFFFF; + + lockdep_assert_held(&kbdev->pm.power_change_lock); + + reg = core_type_to_reg(core_type, action); + + KBASE_DEBUG_ASSERT(reg); +#ifdef CONFIG_MALI_GATOR_SUPPORT + if (cores) { + if (action == ACTION_PWRON) + kbase_trace_mali_pm_power_on(core_type, cores); + else if (action == ACTION_PWROFF) + kbase_trace_mali_pm_power_off(core_type, cores); + } +#endif /* CONFIG_MALI_GATOR_SUPPORT */ + /* Tracing */ + if (cores) { + if (action == ACTION_PWRON) + switch (core_type) { + case KBASE_PM_CORE_SHADER: + KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u, lo); + break; + case KBASE_PM_CORE_TILER: + KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL, NULL, 0u, lo); + break; + case KBASE_PM_CORE_L2: + KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL, 0u, lo); + break; + default: + /* L3 not handled */ + break; + } + else if (action == ACTION_PWROFF) + switch (core_type) { + case KBASE_PM_CORE_SHADER: + KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL, 0u, lo); + break; + case KBASE_PM_CORE_TILER: + KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, NULL, 0u, lo); + break; + case KBASE_PM_CORE_L2: + KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL, 0u, lo); + break; + default: + /* L3 not handled */ + break; + } + } + + if (lo != 0) + kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL); + + if (hi != 0) + kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL); +} + +/** Get information about a core set + * + * This function gets information (chosen by \c action) about a set of cores of a type given by \c core_type. It is a + * static function used by @ref kbase_pm_get_present_cores, @ref kbase_pm_get_active_cores, @ref + * kbase_pm_get_trans_cores and @ref kbase_pm_get_ready_cores. + * + * @param kbdev The kbase device structure of the device + * @param core_type The type of core that the should be queried + * @param action The property of the cores to query + * + * @return A bit mask specifying the state of the cores + */ +static u64 kbase_pm_get_state(kbase_device *kbdev, kbase_pm_core_type core_type, kbasep_pm_action action) +{ + u32 reg; + u32 lo, hi; + + reg = core_type_to_reg(core_type, action); + + KBASE_DEBUG_ASSERT(reg); + + lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL); + hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL); + + return (((u64) hi) << 32) | ((u64) lo); +} + +void kbasep_pm_read_present_cores(kbase_device *kbdev) +{ + kbdev->shader_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_SHADER, ACTION_PRESENT); + kbdev->tiler_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_TILER, ACTION_PRESENT); + kbdev->l2_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_L2, ACTION_PRESENT); + kbdev->l3_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_L3, ACTION_PRESENT); + + kbdev->shader_inuse_bitmap = 0; + kbdev->shader_needed_bitmap = 0; + kbdev->shader_available_bitmap = 0; + kbdev->tiler_available_bitmap = 0; + kbdev->l2_users_count = 0; + kbdev->l2_available_bitmap = 0; + kbdev->tiler_needed_cnt = 0; + kbdev->tiler_inuse_cnt = 0; + + memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt)); +} + +KBASE_EXPORT_TEST_API(kbasep_pm_read_present_cores) + +/** Get the cores that are present + */ +u64 kbase_pm_get_present_cores(kbase_device *kbdev, kbase_pm_core_type type) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + switch (type) { + case KBASE_PM_CORE_L3: + return kbdev->l3_present_bitmap; + break; + case KBASE_PM_CORE_L2: + return kbdev->l2_present_bitmap; + break; + case KBASE_PM_CORE_SHADER: + return kbdev->shader_present_bitmap; + break; + case KBASE_PM_CORE_TILER: + return kbdev->tiler_present_bitmap; + break; + } + KBASE_DEBUG_ASSERT(0); + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores) + +/** Get the cores that are "active" (busy processing work) + */ +u64 kbase_pm_get_active_cores(kbase_device *kbdev, kbase_pm_core_type type) +{ + return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE); +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores) + +/** Get the cores that are transitioning between power states + */ +u64 kbase_pm_get_trans_cores(kbase_device *kbdev, kbase_pm_core_type type) +{ + return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS); +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores) +/** Get the cores that are powered on + */ +u64 kbase_pm_get_ready_cores(kbase_device *kbdev, kbase_pm_core_type type) +{ + u64 result; + result = kbase_pm_get_state(kbdev, type, ACTION_READY); + + switch (type) { + case KBASE_PM_CORE_SHADER: + KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u, (u32) result); + break; + case KBASE_PM_CORE_TILER: + KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u, (u32) result); + break; + case KBASE_PM_CORE_L2: + KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u, (u32) result); + break; + default: + /* NB: L3 not currently traced */ + break; + } + + return result; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores) + +/** Perform power transitions for a particular core type. + * + * This function will perform any available power transitions to make the actual hardware state closer to the desired + * state. If a core is currently transitioning then changes to the power state of that call cannot be made until the + * transition has finished. Cores which are not present in the hardware are ignored if they are specified in the + * desired_state bitmask, however the return value will always be 0 in this case. + * + * @param kbdev The kbase device + * @param type The core type to perform transitions for + * @param desired_state A bit mask of the desired state of the cores + * @param in_use A bit mask of the cores that are currently running jobs. + * These cores have to be kept powered up because there are jobs + * running (or about to run) on them. + * @param[out] available Receives a bit mask of the cores that the job scheduler can use to submit jobs to. + * May be NULL if this is not needed. + * @param[in,out] powering_on Bit mask to update with cores that are transitioning to a power-on state. + * + * @return MALI_TRUE if the desired state has been reached, MALI_FALSE otherwise + */ +STATIC mali_bool kbase_pm_transition_core_type(kbase_device *kbdev, kbase_pm_core_type type, u64 desired_state, + u64 in_use, u64 * const available, u64 *powering_on) +{ + u64 present; + u64 ready; + u64 trans; + u64 powerup; + u64 powerdown; + u64 powering_on_trans; + u64 desired_state_in_use; + + lockdep_assert_held(&kbdev->pm.power_change_lock); + + /* Get current state */ + present = kbase_pm_get_present_cores(kbdev, type); + trans = kbase_pm_get_trans_cores(kbdev, type); + ready = kbase_pm_get_ready_cores(kbdev, type); + + powering_on_trans = trans & *powering_on; + *powering_on = powering_on_trans; + + if (available != NULL) + *available = (ready | powering_on_trans) & desired_state; + + /* Update desired state to include the in-use cores. These have to be kept powered up because there are jobs + * running or about to run on these cores + */ + desired_state_in_use = desired_state | in_use; + + /* Update state of whether l2 caches are powered */ + if (type == KBASE_PM_CORE_L2) { + if ((ready == present) && (desired_state_in_use == ready) && (trans == 0)) { + /* All are ready, none will be turned off, and none are transitioning */ + kbdev->pm.l2_powered = 1; + if (kbdev->l2_users_count > 0) { + /* Notify any registered l2 cache users (optimized out when no users waiting) */ + wake_up(&kbdev->pm.l2_powered_wait); + } + } else { + kbdev->pm.l2_powered = 0; + } + } + + if (desired_state_in_use == ready && (trans == 0)) + return MALI_TRUE; + + /* Restrict the cores to those that are actually present */ + powerup = desired_state_in_use & present; + powerdown = (~desired_state_in_use) & present; + + /* Restrict to cores that are not already in the desired state */ + powerup &= ~ready; + powerdown &= ready; + + /* Don't transition any cores that are already transitioning */ + powerup &= ~trans; + powerdown &= ~trans; + + /* Perform transitions if any */ + kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON); + kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF); + + /* Recalculate cores transitioning on, and re-evaluate our state */ + powering_on_trans |= powerup; + *powering_on = powering_on_trans; + if (available != NULL) + *available = (ready | powering_on_trans) & desired_state; + + return MALI_FALSE; +} + +KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type) + +/** Determine which caches should be on for a particular core state. + * + * This function takes a bit mask of the present caches and the cores (or caches) that are attached to the caches that + * will be powered. It then computes which caches should be turned on to allow the cores requested to be powered up. + * + * @param present The bit mask of present caches + * @param cores_powered A bit mask of cores (or L2 caches) that are desired to be powered + * + * @return A bit mask of the caches that should be turned on + */ +STATIC u64 get_desired_cache_status(u64 present, u64 cores_powered) +{ + u64 desired = 0; + + while (present) { + /* Find out which is the highest set bit */ + u64 bit = fls64(present) - 1; + u64 bit_mask = 1ull << bit; + /* Create a mask which has all bits from 'bit' upwards set */ + + u64 mask = ~(bit_mask - 1); + + /* If there are any cores powered at this bit or above (that haven't previously been processed) then we need + * this core on */ + if (cores_powered & mask) + desired |= bit_mask; + + /* Remove bits from cores_powered and present */ + cores_powered &= ~mask; + present &= ~bit_mask; + } + + return desired; +} + +KBASE_EXPORT_TEST_API(get_desired_cache_status) + +mali_bool MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev) +{ + mali_bool cores_are_available = MALI_FALSE; + mali_bool in_desired_state = MALI_TRUE; + u64 desired_l2_state; + u64 desired_l3_state; + u64 cores_powered; + u64 tiler_available_bitmap; + u64 shader_available_bitmap; + u64 shader_ready_bitmap; + u64 shader_transitioning_bitmap; + u64 l2_available_bitmap; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->pm.power_change_lock); + + spin_lock(&kbdev->pm.gpu_powered_lock); + if (kbdev->pm.gpu_powered == MALI_FALSE) { + spin_unlock(&kbdev->pm.gpu_powered_lock); + if (kbdev->pm.desired_shader_state == 0 && kbdev->pm.desired_tiler_state == 0) + return MALI_TRUE; + return MALI_FALSE; + } + + /* Trace that a change-state is being requested, and that it took + * (effectively) no time to start it. This is useful for counting how many + * state changes occurred, in a way that's backwards-compatible with + * processing the trace data */ + kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); + kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); + + /* If any cores are already powered then, we must keep the caches on */ + cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); + + cores_powered |= kbdev->pm.desired_shader_state; + + /* If there are l2 cache users registered, keep all l2s powered even if all other cores are off. */ + if (kbdev->l2_users_count > 0) + cores_powered |= kbdev->l2_present_bitmap; + + desired_l2_state = get_desired_cache_status(kbdev->l2_present_bitmap, cores_powered); + + /* If any l2 cache is on, then enable l2 #0, for use by job manager */ + if (0 != desired_l2_state) { + desired_l2_state |= 1; + /* Also enable tiler if l2 cache is powered */ + kbdev->pm.desired_tiler_state = kbdev->tiler_present_bitmap; + } else { + kbdev->pm.desired_tiler_state = 0; + } + + desired_l3_state = get_desired_cache_status(kbdev->l3_present_bitmap, desired_l2_state); + + in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_L3, desired_l3_state, 0, NULL, &kbdev->pm.powering_on_l3_state); + in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_L2, desired_l2_state, 0, &l2_available_bitmap, &kbdev->pm.powering_on_l2_state); + + if( kbdev->l2_available_bitmap != l2_available_bitmap) + { + KBASE_TIMELINE_POWER_L2(kbdev,l2_available_bitmap); + } + + kbdev->l2_available_bitmap = l2_available_bitmap; + + if (in_desired_state) { + + in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_TILER, kbdev->pm.desired_tiler_state, 0, &tiler_available_bitmap, &kbdev->pm.powering_on_tiler_state); + in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_SHADER, kbdev->pm.desired_shader_state, kbdev->shader_inuse_bitmap, &shader_available_bitmap, &kbdev->pm.powering_on_shader_state); + + if (kbdev->shader_available_bitmap != shader_available_bitmap) { + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, NULL, 0u, (u32) shader_available_bitmap); + KBASE_TIMELINE_POWER_SHADER(kbdev, shader_available_bitmap); + } + + kbdev->shader_available_bitmap = shader_available_bitmap; + + if (kbdev->tiler_available_bitmap != tiler_available_bitmap) { + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, NULL, 0u, (u32) tiler_available_bitmap); + KBASE_TIMELINE_POWER_TILER(kbdev, tiler_available_bitmap); + } + + kbdev->tiler_available_bitmap = tiler_available_bitmap; + + } else if ((l2_available_bitmap & kbdev->tiler_present_bitmap) != kbdev->tiler_present_bitmap) { + tiler_available_bitmap = 0; + + if (kbdev->tiler_available_bitmap != tiler_available_bitmap) { + KBASE_TIMELINE_POWER_TILER(kbdev, tiler_available_bitmap); + } + + kbdev->tiler_available_bitmap = tiler_available_bitmap; + } + + /* State updated for slow-path waiters */ + kbdev->pm.gpu_in_desired_state = in_desired_state; + + shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); + shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER); + + /* Determine whether the cores are now available (even if the set of + * available cores is empty). Note that they can be available even if we've + * not finished transitioning to the desired state */ + if ((kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state) == kbdev->pm.desired_shader_state + && (kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state) == kbdev->pm.desired_tiler_state) { + cores_are_available = MALI_TRUE; + + KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u, (u32)(kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state)); + KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u, (u32)(kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state)); + + /* Log timelining information about handling events that power up + * cores, to match up either with immediate submission either because + * cores already available, or from PM IRQ */ + if (!in_desired_state) + kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + } + + if (in_desired_state) { + KBASE_DEBUG_ASSERT(cores_are_available); + +#ifdef CONFIG_MALI_GATOR_SUPPORT + kbase_trace_mali_pm_status(KBASE_PM_CORE_L3, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L3)); + kbase_trace_mali_pm_status(KBASE_PM_CORE_L2, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2)); + kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER)); + kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER)); +#endif /* CONFIG_MALI_GATOR_SUPPORT */ + + KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL, kbdev->pm.gpu_in_desired_state, (u32)kbdev->pm.desired_shader_state); + KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u, (u32)kbdev->pm.desired_tiler_state); + + /* Log timelining information for synchronous waiters */ + kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + /* Wake slow-path waiters. Job scheduler does not use this. */ + KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0); + wake_up(&kbdev->pm.gpu_in_desired_state_wait); + } + + spin_unlock(&kbdev->pm.gpu_powered_lock); + + /* kbase_pm_ca_update_core_status can cause one-level recursion into + * this function, so it must only be called once all changes to kbdev + * have been committed, and after the gpu_powered_lock has been + * dropped. */ + if (kbdev->shader_ready_bitmap != shader_ready_bitmap || + kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) { + kbdev->shader_ready_bitmap = shader_ready_bitmap; + kbdev->shader_transitioning_bitmap = shader_transitioning_bitmap; + + kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap, shader_transitioning_bitmap); + } + + /* The core availability policy is not allowed to keep core group 0 off */ + if (!((shader_ready_bitmap | shader_transitioning_bitmap) & kbdev->gpu_props.props.coherency_info.group[0].core_mask) && + !(kbase_pm_ca_get_core_mask(kbdev) & kbdev->gpu_props.props.coherency_info.group[0].core_mask)) + BUG(); + + /* The core availability policy is allowed to keep core group 1 off, + * but all jobs specifically targeting CG1 must fail */ + if (!((shader_ready_bitmap | shader_transitioning_bitmap) & kbdev->gpu_props.props.coherency_info.group[1].core_mask) && + !(kbase_pm_ca_get_core_mask(kbdev) & kbdev->gpu_props.props.coherency_info.group[1].core_mask)) + kbdev->pm.cg1_disabled = MALI_TRUE; + else + kbdev->pm.cg1_disabled = MALI_FALSE; + + return cores_are_available; +} +KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock) + +void kbase_pm_check_transitions_sync(struct kbase_device *kbdev) +{ + unsigned long flags; + mali_bool cores_are_available; + /* Force the transition to be checked and reported - the cores may be + * 'available' (for job submission) but not fully powered up. */ + spin_lock_irqsave(&kbdev->pm.power_change_lock, flags); + cores_are_available = kbase_pm_check_transitions_nolock(kbdev); + /* Don't need 'cores_are_available', because we don't return anything */ + CSTD_UNUSED(cores_are_available); + spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags); + + /* Wait for cores */ + wait_event(kbdev->pm.gpu_in_desired_state_wait, kbdev->pm.gpu_in_desired_state); + + /* Log timelining information that a change in state has completed */ + kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); +} +KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync) + +void kbase_pm_enable_interrupts(kbase_device *kbdev) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + /* + * Clear all interrupts, + * and unmask them all. + */ + spin_lock_irqsave(&kbdev->pm.power_change_lock, flags); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL, NULL); + spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags); + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, NULL); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL); +} + +KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts) + +void kbase_pm_disable_interrupts(kbase_device *kbdev) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + /* + * Mask all interrupts, + * and clear them all. + */ + spin_lock_irqsave(&kbdev->pm.power_change_lock, flags); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, NULL); + spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags); + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, NULL); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); +} + +KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts) + +/* + * pmu layout: + * 0x0000: PMU TAG (RO) (0xCAFECAFE) + * 0x0004: PMU VERSION ID (RO) (0x00000000) + * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE) + */ +void kbase_pm_clock_on(kbase_device *kbdev) +{ + unsigned long flags; + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->pm.lock); + + if (kbdev->pm.gpu_powered) { + /* Already turned on */ + return; + } + + KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u); + + if (kbdev->pm.callback_power_on && kbdev->pm.callback_power_on(kbdev)) { + /* GPU state was lost, reset GPU to ensure it is in a consistent state */ + kbase_pm_init_hw(kbdev,MALI_TRUE); + } + + spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags); + kbdev->pm.gpu_powered = MALI_TRUE; + spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags); + + /* Lastly, enable the interrupts */ + kbase_pm_enable_interrupts(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_clock_on) + +void kbase_pm_clock_off(kbase_device *kbdev) +{ + unsigned long flags; + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->pm.lock); + + /* ASSERT that the cores should now be unavailable. No lock needed. */ + KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u); + + if (!kbdev->pm.gpu_powered) { + /* Already turned off */ + return; + } + + KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u); + + /* Disable interrupts. This also clears any outstanding interrupts */ + kbase_pm_disable_interrupts(kbdev); + /* Ensure that any IRQ handlers have finished */ + kbase_synchronize_irqs(kbdev); + + /* The GPU power may be turned off from this point */ + spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags); + kbdev->pm.gpu_powered = MALI_FALSE; + spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags); + + if (kbdev->pm.callback_power_off) + kbdev->pm.callback_power_off(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_clock_off) + +struct kbasep_reset_timeout_data { + struct hrtimer timer; + mali_bool timed_out; + kbase_device *kbdev; +}; + +void kbase_pm_reset_done(kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + kbdev->pm.reset_done = MALI_TRUE; + wake_up(&kbdev->pm.reset_done_wait); +} + +/** + * Wait for the RESET_COMPLETED IRQ to occur, then reset the waiting state. + */ +STATIC void kbase_pm_wait_for_reset(kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->pm.lock); + + wait_event(kbdev->pm.reset_done_wait, (kbdev->pm.reset_done)); + kbdev->pm.reset_done = MALI_FALSE; +} + +KBASE_EXPORT_TEST_API(kbase_pm_reset_done) + +static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) +{ + struct kbasep_reset_timeout_data *rtdata = container_of(timer, struct kbasep_reset_timeout_data, timer); + + rtdata->timed_out = 1; + + /* Set the wait queue to wake up kbase_pm_init_hw even though the reset hasn't completed */ + kbase_pm_reset_done(rtdata->kbdev); + + return HRTIMER_NORESTART; +} + +static void kbase_pm_hw_issues(kbase_device *kbdev) +{ + u32 value = 0; + u32 config_value; + + /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443. */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443)) + value |= (1 << 16); + + /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327. */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327)) + value |= (1 << 6); + + /* Enable alternative hardware counter selection if configured. */ + if( kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_ALTERNATIVE_HWC) ) + value |= (1 << 3); + + if (value != 0) + kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), value, NULL); + + /* Limit the GPU bus bandwidth if the platform needs this. */ + value = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL); + + /* Limit read ID width for AXI */ + config_value = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_ARID_LIMIT); + value &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS); + value |= (config_value & 0x3) << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT; + + /* Limit write ID width for AXI */ + config_value = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_AWID_LIMIT); + value &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES); + value |= (config_value & 0x3) << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT; + + kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), value, NULL); +} + +mali_error kbase_pm_init_hw(kbase_device *kbdev, mali_bool enable_irqs ) +{ + unsigned long flags; + struct kbasep_reset_timeout_data rtdata; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->pm.lock); + + /* Ensure the clock is on before attempting to access the hardware */ + if (!kbdev->pm.gpu_powered) { + if (kbdev->pm.callback_power_on) + kbdev->pm.callback_power_on(kbdev); + + spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags); + kbdev->pm.gpu_powered = MALI_TRUE; + spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags); + } + + /* Ensure interrupts are off to begin with, this also clears any outstanding interrupts */ + kbase_pm_disable_interrupts(kbdev); + + /* Prepare for the soft-reset */ + kbdev->pm.reset_done = MALI_FALSE; + + /* The cores should be made unavailable due to the reset */ + spin_lock_irqsave(&kbdev->pm.power_change_lock, flags); + if (kbdev->shader_available_bitmap != 0u) + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, NULL, 0u, (u32)0u); + if (kbdev->tiler_available_bitmap != 0u) + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, NULL, 0u, (u32)0u); + kbdev->shader_available_bitmap = 0u; + kbdev->tiler_available_bitmap = 0u; + kbdev->l2_available_bitmap = 0u; + spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags); + + /* Soft reset the GPU */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_SOFT_RESET, NULL); + + /* Unmask the reset complete interrupt only */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED, NULL); + + /* Initialize a structure for tracking the status of the reset */ + rtdata.kbdev = kbdev; + rtdata.timed_out = 0; + + /* Create a timer to use as a timeout on the reset */ + hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rtdata.timer.function = kbasep_reset_timeout; + + hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), HRTIMER_MODE_REL); + + /* Wait for the RESET_COMPLETED interrupt to be raised */ + kbase_pm_wait_for_reset(kbdev); + + if (rtdata.timed_out == 0) { + /* GPU has been reset */ + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + goto out; + } + + /* No interrupt has been received - check if the RAWSTAT register says the reset has completed */ + if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & RESET_COMPLETED) { + /* The interrupt is set in the RAWSTAT; this suggests that the interrupts are not getting to the CPU */ + KBASE_DEBUG_PRINT_WARN(KBASE_PM, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); + /* If interrupts aren't working we can't continue. */ + destroy_hrtimer_on_stack(&rtdata.timer); + goto out; + } + + /* The GPU doesn't seem to be responding to the reset so try a hard reset */ + KBASE_DEBUG_PRINT_ERROR(KBASE_PM, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", RESET_TIMEOUT); + KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET, NULL); + + /* Restart the timer to wait for the hard reset to complete */ + rtdata.timed_out = 0; + + hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), HRTIMER_MODE_REL); + + /* Wait for the RESET_COMPLETED interrupt to be raised */ + kbase_pm_wait_for_reset(kbdev); + + if (rtdata.timed_out == 0) { + /* GPU has been reset */ + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + goto out; + } + + destroy_hrtimer_on_stack(&rtdata.timer); + + KBASE_DEBUG_PRINT_ERROR(KBASE_PM, "Failed to hard-reset the GPU (timed out after %d ms)\n", RESET_TIMEOUT); + + /* The GPU still hasn't reset, give up */ + return MALI_ERROR_FUNCTION_FAILED; + + out: + /* Re-enable interrupts if requested*/ + if ( enable_irqs ) + { + kbase_pm_enable_interrupts(kbdev); + } + /* If cycle counter was in use-re enable it */ + spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags); + + if (kbdev->pm.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_START, NULL); + + spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags); + + kbase_pm_hw_issues(kbdev); + + return MALI_ERROR_NONE; +} + +KBASE_EXPORT_TEST_API(kbase_pm_init_hw) + +void kbase_pm_request_gpu_cycle_counter(kbase_device *kbdev) +{ + unsigned long flags; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + KBASE_DEBUG_ASSERT(kbdev->pm.gpu_powered); + + spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags); + + KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests < INT_MAX); + + ++kbdev->pm.gpu_cycle_counter_requests; + + if (1 == kbdev->pm.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_START, NULL); + + spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter) + +void kbase_pm_release_gpu_cycle_counter(kbase_device *kbdev) +{ + unsigned long flags; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags); + + KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests > 0); + + --kbdev->pm.gpu_cycle_counter_requests; + + if (0 == kbdev->pm.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL); + + spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags); +} + +KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter) |