aboutsummaryrefslogtreecommitdiff
path: root/drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_instr.c
diff options
context:
space:
mode:
authorShow Liu <show.liu@linaro.org>2013-12-25 14:14:27 +0800
committerShow Liu <show.liu@linaro.org>2014-04-28 11:37:14 +0800
commit2fdaee08fcbc16be5e6cd984017deffd10533b2f (patch)
tree373571b0de6134a89ab0924685e355fd1e91afcc /drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_instr.c
parent87b98c1fa196a2d41af2987165b53a921322f3d0 (diff)
downloadlinaro-lsk-2fdaee08fcbc16be5e6cd984017deffd10533b2f.tar.gz
Added Mali T6xx GPU driver
Diffstat (limited to 'drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_instr.c')
-rwxr-xr-xdrivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_instr.c616
1 files changed, 616 insertions, 0 deletions
diff --git a/drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_instr.c b/drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_instr.c
new file mode 100755
index 00000000000..484482065ae
--- /dev/null
+++ b/drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_instr.c
@@ -0,0 +1,616 @@
+/*
+ *
+ * (C) COPYRIGHT 2011-2013 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/**
+ * @file mali_kbase_instr.c
+ * Base kernel instrumentation APIs.
+ */
+
+#include <kbase/src/common/mali_kbase.h>
+#include <kbase/src/common/mali_midg_regmap.h>
+
+/**
+ * @brief Issue Cache Clean & Invalidate command to hardware
+ */
+static void kbasep_instr_hwcnt_cacheclean(kbase_device *kbdev)
+{
+ unsigned long flags;
+ unsigned long pm_flags;
+ u32 irq_mask;
+
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ /* Wait for any reset to complete */
+ while (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.cache_clean_wait,
+ kbdev->hwcnt.state != KBASE_INSTR_STATE_RESETTING);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_REQUEST_CLEAN);
+
+ /* Enable interrupt */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | CLEAN_CACHES_COMPLETED, NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
+
+ /* clean&invalidate the caches so we're sure the mmu tables for the dump buffer is valid */
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CLEAN_INV_CACHES, NULL);
+ kbdev->hwcnt.state = KBASE_INSTR_STATE_CLEANING;
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+}
+
+STATIC mali_error kbase_instr_hwcnt_enable_internal(kbase_device *kbdev, kbase_context *kctx, kbase_uk_hwcnt_setup *setup)
+{
+ unsigned long flags, pm_flags;
+ mali_error err = MALI_ERROR_FUNCTION_FAILED;
+ kbasep_js_device_data *js_devdata;
+ u32 irq_mask;
+ int ret;
+ u64 shader_cores_needed;
+
+ KBASE_DEBUG_ASSERT(NULL != kctx);
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+ KBASE_DEBUG_ASSERT(NULL != setup);
+ KBASE_DEBUG_ASSERT(NULL == kbdev->hwcnt.suspended_kctx);
+
+ shader_cores_needed = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER);
+
+ js_devdata = &kbdev->js_data;
+
+ /* alignment failure */
+ if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1)))
+ goto out_err;
+
+ /* Override core availability policy to ensure all cores are available */
+ kbase_pm_ca_instr_enable(kbdev);
+
+ /* Mark the context as active so the GPU is kept turned on */
+ /* A suspend won't happen here, because we're in a syscall from a userspace
+ * thread. */
+ kbase_pm_context_active(kbdev);
+
+ /* Request the cores early on synchronously - we'll release them on any errors
+ * (e.g. instrumentation already active) */
+ kbase_pm_request_cores_sync(kbdev, MALI_TRUE, shader_cores_needed);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
+ /* GPU is being reset */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+
+ if (kbdev->hwcnt.state != KBASE_INSTR_STATE_DISABLED) {
+ /* Instrumentation is already enabled */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ goto out_unrequest_cores;
+ }
+
+ /* Enable interrupt */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | PRFCNT_SAMPLE_COMPLETED, NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
+
+ /* In use, this context is the owner */
+ kbdev->hwcnt.kctx = kctx;
+ /* Remember the dump address so we can reprogram it later */
+ kbdev->hwcnt.addr = setup->dump_buffer;
+ /* Remember all the settings for suspend/resume */
+ if (&kbdev->hwcnt.suspended_state != setup)
+ memcpy(&kbdev->hwcnt.suspended_state, setup, sizeof(kbdev->hwcnt.suspended_state));
+
+ /* Request the clean */
+ kbdev->hwcnt.state = KBASE_INSTR_STATE_REQUEST_CLEAN;
+ kbdev->hwcnt.triggered = 0;
+ /* Clean&invalidate the caches so we're sure the mmu tables for the dump buffer is valid */
+ ret = queue_work(kbdev->hwcnt.cache_clean_wq, &kbdev->hwcnt.cache_clean_work);
+ KBASE_DEBUG_ASSERT(ret);
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ /* Wait for cacheclean to complete */
+ wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
+
+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE);
+
+ /* Schedule the context in */
+ kbasep_js_schedule_privileged_ctx(kbdev, kctx);
+
+ /* Configure */
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_OFF, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), setup->dump_buffer & 0xFFFFFFFF, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), setup->dump_buffer >> 32, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), setup->jm_bm, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), setup->shader_bm, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_L3_CACHE_EN), setup->l3_cache_bm, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), setup->mmu_l2_bm, kctx);
+ /* Due to PRLAM-8186 we need to disable the Tiler before we enable the HW counter dump. */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, kctx);
+ else
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), setup->tiler_bm, kctx);
+
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, kctx);
+
+ /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), setup->tiler_bm, kctx);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
+ /* GPU is being reset */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+
+ kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
+ kbdev->hwcnt.triggered = 1;
+ wake_up(&kbdev->hwcnt.wait);
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ err = MALI_ERROR_NONE;
+
+ KBASE_DEBUG_PRINT_INFO(KBASE_CORE, "HW counters dumping set-up for context %p", kctx);
+ return err;
+ out_unrequest_cores:
+ kbase_pm_unrequest_cores(kbdev, MALI_TRUE, shader_cores_needed);
+ kbase_pm_context_idle(kbdev);
+ out_err:
+ return err;
+}
+
+/**
+ * @brief Enable HW counters collection
+ *
+ * Note: will wait for a cache clean to complete
+ */
+mali_error kbase_instr_hwcnt_enable(kbase_context *kctx, kbase_uk_hwcnt_setup *setup)
+{
+ kbase_device *kbdev;
+ mali_bool access_allowed;
+ kbdev = kctx->kbdev;
+
+ KBASE_DEBUG_ASSERT(NULL != kctx);
+ /* Determine if the calling task has access to this capability */
+ access_allowed = kbase_security_has_capability(kctx, KBASE_SEC_INSTR_HW_COUNTERS_COLLECT, KBASE_SEC_FLAG_NOAUDIT);
+ if (MALI_FALSE == access_allowed)
+ return MALI_ERROR_FUNCTION_FAILED;
+
+ return kbase_instr_hwcnt_enable_internal(kbdev, kctx, setup);
+}
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_enable)
+
+/**
+ * @brief Disable HW counters collection
+ *
+ * Note: might sleep, waiting for an ongoing dump to complete
+ */
+mali_error kbase_instr_hwcnt_disable(kbase_context *kctx)
+{
+ unsigned long flags, pm_flags;
+ mali_error err = MALI_ERROR_FUNCTION_FAILED;
+ u32 irq_mask;
+ kbase_device *kbdev;
+
+ KBASE_DEBUG_ASSERT(NULL != kctx);
+ kbdev = kctx->kbdev;
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+
+ while (1) {
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.state == KBASE_INSTR_STATE_DISABLED) {
+ /* Instrumentation is not enabled */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ goto out;
+ }
+
+ if (kbdev->hwcnt.kctx != kctx) {
+ /* Instrumentation has been setup for another context */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ goto out;
+ }
+
+ if (kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE)
+ break;
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ /* Ongoing dump/setup - wait for its completion */
+ wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
+
+ }
+
+ kbdev->hwcnt.state = KBASE_INSTR_STATE_DISABLED;
+ kbdev->hwcnt.triggered = 0;
+
+ /* Disable interrupt */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
+
+ /* Disable the counters */
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx);
+
+ kbdev->hwcnt.kctx = NULL;
+ kbdev->hwcnt.addr = 0ULL;
+
+ kbase_pm_ca_instr_disable(kbdev);
+
+ kbase_pm_unrequest_cores(kbdev, MALI_TRUE, kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER));
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ /* Release the context. This had its own Power Manager Active reference */
+ kbasep_js_release_privileged_ctx(kbdev, kctx);
+
+ /* Also release our Power Manager Active reference */
+ kbase_pm_context_idle(kbdev);
+
+ KBASE_DEBUG_PRINT_INFO(KBASE_CORE, "HW counters dumping disabled for context %p", kctx);
+
+ err = MALI_ERROR_NONE;
+
+ out:
+ return err;
+}
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_disable)
+
+/**
+ * @brief Configure HW counters collection
+ */
+mali_error kbase_instr_hwcnt_setup(kbase_context *kctx, kbase_uk_hwcnt_setup *setup)
+{
+ mali_error err = MALI_ERROR_FUNCTION_FAILED;
+ kbase_device *kbdev;
+
+ KBASE_DEBUG_ASSERT(NULL != kctx);
+
+ kbdev = kctx->kbdev;
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+
+ if (NULL == setup) {
+ /* Bad parameter - abort */
+ goto out;
+ }
+
+ if (setup->dump_buffer != 0ULL) {
+ /* Enable HW counters */
+ err = kbase_instr_hwcnt_enable(kctx, setup);
+ } else {
+ /* Disable HW counters */
+ err = kbase_instr_hwcnt_disable(kctx);
+ }
+
+ out:
+ return err;
+}
+
+/**
+ * @brief Issue Dump command to hardware
+ *
+ * Notes:
+ * - does not sleep
+ */
+mali_error kbase_instr_hwcnt_dump_irq(kbase_context *kctx)
+{
+ unsigned long flags;
+ mali_error err = MALI_ERROR_FUNCTION_FAILED;
+ kbase_device *kbdev;
+
+ KBASE_DEBUG_ASSERT(NULL != kctx);
+ kbdev = kctx->kbdev;
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.kctx != kctx) {
+ /* The instrumentation has been setup for another context */
+ goto unlock;
+ }
+
+ if (kbdev->hwcnt.state != KBASE_INSTR_STATE_IDLE) {
+ /* HW counters are disabled or another dump is ongoing, or we're resetting */
+ goto unlock;
+ }
+
+ kbdev->hwcnt.triggered = 0;
+
+ /* Mark that we're dumping - the PF handler can signal that we faulted */
+ kbdev->hwcnt.state = KBASE_INSTR_STATE_DUMPING;
+
+ /* Reconfigure the dump address */
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), kbdev->hwcnt.addr & 0xFFFFFFFF, NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), kbdev->hwcnt.addr >> 32, NULL);
+
+ /* Start dumping */
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL, kbdev->hwcnt.addr, 0);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_SAMPLE, kctx);
+
+ KBASE_DEBUG_PRINT_INFO(KBASE_CORE, "HW counters dumping done for context %p", kctx);
+
+ err = MALI_ERROR_NONE;
+
+ unlock:
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ return err;
+}
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_irq)
+
+/**
+ * @brief Tell whether the HW counters dump has completed
+ *
+ * Notes:
+ * - does not sleep
+ * - success will be set to MALI_TRUE if the dump succeeded or
+ * MALI_FALSE on failure
+ */
+mali_bool kbase_instr_hwcnt_dump_complete(kbase_context *kctx, mali_bool * const success)
+{
+ unsigned long flags;
+ mali_bool complete = MALI_FALSE;
+ kbase_device *kbdev;
+
+ KBASE_DEBUG_ASSERT(NULL != kctx);
+ kbdev = kctx->kbdev;
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+ KBASE_DEBUG_ASSERT(NULL != success);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE) {
+ *success = MALI_TRUE;
+ complete = MALI_TRUE;
+ } else if (kbdev->hwcnt.state == KBASE_INSTR_STATE_FAULT) {
+ *success = MALI_FALSE;
+ complete = MALI_TRUE;
+ kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
+ }
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ return complete;
+}
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete)
+
+/**
+ * @brief Issue Dump command to hardware and wait for completion
+ */
+mali_error kbase_instr_hwcnt_dump(kbase_context *kctx)
+{
+ unsigned long flags;
+ mali_error err = MALI_ERROR_FUNCTION_FAILED;
+ kbase_device *kbdev;
+
+ KBASE_DEBUG_ASSERT(NULL != kctx);
+ kbdev = kctx->kbdev;
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+
+ err = kbase_instr_hwcnt_dump_irq(kctx);
+ if (MALI_ERROR_NONE != err) {
+ /* Can't dump HW counters */
+ goto out;
+ }
+
+ /* Wait for dump & cacheclean to complete */
+ wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
+ /* GPU is being reset */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+
+ if (kbdev->hwcnt.state == KBASE_INSTR_STATE_FAULT) {
+ err = MALI_ERROR_FUNCTION_FAILED;
+ kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
+ } else {
+ /* Dump done */
+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE);
+ err = MALI_ERROR_NONE;
+ }
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ out:
+ return err;
+}
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump)
+
+/**
+ * @brief Clear the HW counters
+ */
+mali_error kbase_instr_hwcnt_clear(kbase_context *kctx)
+{
+ unsigned long flags;
+ mali_error err = MALI_ERROR_FUNCTION_FAILED;
+ kbase_device *kbdev;
+
+ KBASE_DEBUG_ASSERT(NULL != kctx);
+ kbdev = kctx->kbdev;
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
+ /* GPU is being reset */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+
+ /* Check it's the context previously set up and we're not already dumping */
+ if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.state != KBASE_INSTR_STATE_IDLE)
+ goto out;
+
+ /* Clear the counters */
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_CLEAR, kctx);
+
+ err = MALI_ERROR_NONE;
+
+ out:
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ return err;
+}
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear)
+
+/**
+ * Workqueue for handling cache cleaning
+ */
+void kbasep_cache_clean_worker(struct work_struct *data)
+{
+ kbase_device *kbdev;
+ unsigned long flags;
+
+ kbdev = container_of(data, kbase_device, hwcnt.cache_clean_work);
+
+ mutex_lock(&kbdev->cacheclean_lock);
+ kbasep_instr_hwcnt_cacheclean(kbdev);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ /* Wait for our condition, and any reset to complete */
+ while (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING
+ || kbdev->hwcnt.state == KBASE_INSTR_STATE_CLEANING) {
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.cache_clean_wait,
+ (kbdev->hwcnt.state != KBASE_INSTR_STATE_RESETTING
+ && kbdev->hwcnt.state != KBASE_INSTR_STATE_CLEANING));
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_CLEANED);
+
+ /* All finished and idle */
+ kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
+ kbdev->hwcnt.triggered = 1;
+ wake_up(&kbdev->hwcnt.wait);
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ mutex_unlock(&kbdev->cacheclean_lock);
+}
+
+/**
+ * @brief Dump complete interrupt received
+ */
+void kbase_instr_hwcnt_sample_done(kbase_device *kbdev)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.state == KBASE_INSTR_STATE_FAULT) {
+ kbdev->hwcnt.triggered = 1;
+ wake_up(&kbdev->hwcnt.wait);
+ } else if (kbdev->hwcnt.state == KBASE_INSTR_STATE_DUMPING) {
+ int ret;
+ /* Always clean and invalidate the cache after a successful dump */
+ kbdev->hwcnt.state = KBASE_INSTR_STATE_REQUEST_CLEAN;
+ ret = queue_work(kbdev->hwcnt.cache_clean_wq, &kbdev->hwcnt.cache_clean_work);
+ KBASE_DEBUG_ASSERT(ret);
+ }
+ /* NOTE: In the state KBASE_INSTR_STATE_RESETTING, We're in a reset,
+ * and the instrumentation state hasn't been restored yet -
+ * kbasep_reset_timeout_worker() will do the rest of the work */
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+}
+
+/**
+ * @brief Cache clean interrupt received
+ */
+void kbase_clean_caches_done(kbase_device *kbdev)
+{
+ u32 irq_mask;
+
+ if (kbdev->hwcnt.state != KBASE_INSTR_STATE_DISABLED) {
+ unsigned long flags;
+ unsigned long pm_flags;
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ /* Disable interrupt */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask & ~CLEAN_CACHES_COMPLETED, NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
+
+ /* Wakeup... */
+ if (kbdev->hwcnt.state == KBASE_INSTR_STATE_CLEANING) {
+ /* Only wake if we weren't resetting */
+ kbdev->hwcnt.state = KBASE_INSTR_STATE_CLEANED;
+ wake_up(&kbdev->hwcnt.cache_clean_wait);
+ }
+ /* NOTE: In the state KBASE_INSTR_STATE_RESETTING, We're in a reset,
+ * and the instrumentation state hasn't been restored yet -
+ * kbasep_reset_timeout_worker() will do the rest of the work */
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ }
+}
+
+
+/* Disable instrumentation and wait for any existing dump to complete
+ * It's assumed that there's only one privileged context
+ * Safe to do this without lock when doing an OS suspend, because it only
+ * changes in response to user-space IOCTLs */
+void kbase_instr_hwcnt_suspend(kbase_device *kbdev)
+{
+ kbase_context *kctx;
+ KBASE_DEBUG_ASSERT(kbdev);
+ KBASE_DEBUG_ASSERT(!kbdev->hwcnt.suspended_kctx);
+
+ kctx = kbdev->hwcnt.kctx;
+ kbdev->hwcnt.suspended_kctx = kctx;
+
+ /* Relevant state was saved into hwcnt.suspended_state when enabling the
+ * counters */
+
+ if (kctx)
+ {
+ KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_PRIVILEGED);
+ kbase_instr_hwcnt_disable(kctx);
+ }
+}
+
+void kbase_instr_hwcnt_resume(kbase_device *kbdev)
+{
+ kbase_context *kctx;
+ KBASE_DEBUG_ASSERT(kbdev);
+
+ kctx = kbdev->hwcnt.suspended_kctx;
+ kbdev->hwcnt.suspended_kctx = NULL;
+
+ if (kctx)
+ {
+ mali_error err;
+ err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &kbdev->hwcnt.suspended_state);
+ WARN(err != MALI_ERROR_NONE,
+ "Failed to restore instrumented hardware counters on resume\n");
+ }
+}
+