summaryrefslogtreecommitdiff
path: root/drivers/gpu/arm/utgard/linux
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/arm/utgard/linux')
-rw-r--r--drivers/gpu/arm/utgard/linux/license/gpl/mali_kernel_license.h30
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_device_pause_resume.c36
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_kernel_linux.c941
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_kernel_linux.h30
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.c1410
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.h29
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_linux_trace.h162
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory.c510
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory.h143
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.c362
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.h58
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_cow.c754
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_cow.h48
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.c246
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.h65
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.c369
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.h53
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_external.c91
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_external.h29
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_manager.c965
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_manager.h51
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.c805
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.h54
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.c942
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.h121
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_types.h208
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_ump.c154
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_ump.h29
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_util.c149
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_util.h20
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_virtual.c127
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_memory_virtual.h35
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_atomics.c59
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_bitmap.c152
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_irq.c200
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_locks.c287
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_locks.h326
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_low_level_mem.c146
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_mali.c390
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_math.c27
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_memory.c61
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_misc.c92
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_notification.c182
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_pm.c83
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_profiling.c1272
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_specific.h72
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_time.c59
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_timers.c76
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_wait_queue.c78
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_osk_wq.c240
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_pmu_power_up_down.c23
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_profiling_events.h17
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_profiling_gator_api.h17
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_profiling_internal.c275
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_profiling_internal.h35
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_sync.c447
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_sync.h111
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_uk_types.h17
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_ukk_core.c146
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_ukk_gp.c91
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_ukk_mem.c333
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_ukk_pp.c105
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_ukk_profiling.c177
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_ukk_soft_job.c90
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_ukk_timeline.c88
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_ukk_vsync.c39
-rw-r--r--drivers/gpu/arm/utgard/linux/mali_ukk_wrappers.h75
67 files changed, 14914 insertions, 0 deletions
diff --git a/drivers/gpu/arm/utgard/linux/license/gpl/mali_kernel_license.h b/drivers/gpu/arm/utgard/linux/license/gpl/mali_kernel_license.h
new file mode 100644
index 000000000000..c88c992e859b
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/license/gpl/mali_kernel_license.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2010, 2013, 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_kernel_license.h
+ * Defines for the macro MODULE_LICENSE.
+ */
+
+#ifndef __MALI_KERNEL_LICENSE_H__
+#define __MALI_KERNEL_LICENSE_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MALI_KERNEL_LINUX_LICENSE "GPL"
+#define MALI_LICENSE_IS_GPL 1
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __MALI_KERNEL_LICENSE_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_device_pause_resume.c b/drivers/gpu/arm/utgard/linux/mali_device_pause_resume.c
new file mode 100644
index 000000000000..37076a2eaf67
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_device_pause_resume.c
@@ -0,0 +1,36 @@
+/**
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_device_pause_resume.c
+ * Implementation of the Mali pause/resume functionality
+ */
+
+#include <linux/module.h>
+#include <linux/mali/mali_utgard.h>
+#include "mali_pm.h"
+
+void mali_dev_pause(void)
+{
+ /*
+ * Deactive all groups to prevent hardware being touched
+ * during the period of mali device pausing
+ */
+ mali_pm_os_suspend(MALI_FALSE);
+}
+
+EXPORT_SYMBOL(mali_dev_pause);
+
+void mali_dev_resume(void)
+{
+ mali_pm_os_resume();
+}
+
+EXPORT_SYMBOL(mali_dev_resume);
diff --git a/drivers/gpu/arm/utgard/linux/mali_kernel_linux.c b/drivers/gpu/arm/utgard/linux/mali_kernel_linux.c
new file mode 100644
index 000000000000..34cb3d7ca95a
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_kernel_linux.c
@@ -0,0 +1,941 @@
+/**
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_kernel_linux.c
+ * Implementation of the Linux device driver entrypoints
+ */
+#include <linux/module.h> /* kernel module definitions */
+#include <linux/fs.h> /* file system operations */
+#include <linux/cdev.h> /* character device definitions */
+#include <linux/mm.h> /* memory manager definitions */
+#include <linux/mali/mali_utgard_ioctl.h>
+#include <linux/version.h>
+#include <linux/device.h>
+#include "mali_kernel_license.h"
+#include <linux/platform_device.h>
+#include <linux/miscdevice.h>
+#include <linux/bug.h>
+#include <linux/of.h>
+
+#include <linux/mali/mali_utgard.h>
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_kernel_core.h"
+#include "mali_osk.h"
+#include "mali_kernel_linux.h"
+#include "mali_ukk.h"
+#include "mali_ukk_wrappers.h"
+#include "mali_kernel_sysfs.h"
+#include "mali_pm.h"
+#include "mali_kernel_license.h"
+#include "mali_memory.h"
+#include "mali_memory_dma_buf.h"
+#include "mali_memory_manager.h"
+#include "mali_memory_swap_alloc.h"
+#if defined(CONFIG_MALI400_INTERNAL_PROFILING)
+#include "mali_profiling_internal.h"
+#endif
+#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
+#include "mali_osk_profiling.h"
+#include "mali_dvfs_policy.h"
+
+static int is_first_resume = 1;
+/*Store the clk and vol for boot/insmod and mali_resume*/
+static struct mali_gpu_clk_item mali_gpu_clk[2];
+#endif
+
+/* Streamline support for the Mali driver */
+#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_MALI400_PROFILING)
+/* Ask Linux to create the tracepoints */
+#define CREATE_TRACE_POINTS
+#include "mali_linux_trace.h"
+
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_event);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_hw_counter);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_sw_counters);
+#endif /* CONFIG_TRACEPOINTS */
+
+/* from the __malidrv_build_info.c file that is generated during build */
+extern const char *__malidrv_build_info(void);
+
+/* Module parameter to control log level */
+int mali_debug_level = 2;
+module_param(mali_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */
+MODULE_PARM_DESC(mali_debug_level, "Higher number, more dmesg output");
+
+extern int mali_max_job_runtime;
+module_param(mali_max_job_runtime, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_max_job_runtime, "Maximum allowed job runtime in msecs.\nJobs will be killed after this no matter what");
+
+extern int mali_l2_max_reads;
+module_param(mali_l2_max_reads, int, S_IRUSR | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_l2_max_reads, "Maximum reads for Mali L2 cache");
+
+extern unsigned int mali_dedicated_mem_start;
+module_param(mali_dedicated_mem_start, uint, S_IRUSR | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_dedicated_mem_start, "Physical start address of dedicated Mali GPU memory.");
+
+extern unsigned int mali_dedicated_mem_size;
+module_param(mali_dedicated_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_dedicated_mem_size, "Size of dedicated Mali GPU memory.");
+
+extern unsigned int mali_shared_mem_size;
+module_param(mali_shared_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_shared_mem_size, "Size of shared Mali GPU memory.");
+
+#if defined(CONFIG_MALI400_PROFILING)
+extern int mali_boot_profiling;
+module_param(mali_boot_profiling, int, S_IRUSR | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_boot_profiling, "Start profiling as a part of Mali driver initialization");
+#endif
+
+extern int mali_max_pp_cores_group_1;
+module_param(mali_max_pp_cores_group_1, int, S_IRUSR | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_max_pp_cores_group_1, "Limit the number of PP cores to use from first PP group.");
+
+extern int mali_max_pp_cores_group_2;
+module_param(mali_max_pp_cores_group_2, int, S_IRUSR | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_max_pp_cores_group_2, "Limit the number of PP cores to use from second PP group (Mali-450 only).");
+
+extern unsigned int mali_mem_swap_out_threshold_value;
+module_param(mali_mem_swap_out_threshold_value, uint, S_IRUSR | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_mem_swap_out_threshold_value, "Threshold value used to limit how much swappable memory cached in Mali driver.");
+
+#if defined(CONFIG_MALI_DVFS)
+/** the max fps the same as display vsync default 60, can set by module insert parameter */
+extern int mali_max_system_fps;
+module_param(mali_max_system_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_max_system_fps, "Max system fps the same as display VSYNC.");
+
+/** a lower limit on their desired FPS default 58, can set by module insert parameter*/
+extern int mali_desired_fps;
+module_param(mali_desired_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
+MODULE_PARM_DESC(mali_desired_fps, "A bit lower than max_system_fps which user desired fps");
+#endif
+
+#if MALI_ENABLE_CPU_CYCLES
+#include <linux/cpumask.h>
+#include <linux/timer.h>
+#include <asm/smp.h>
+static struct timer_list mali_init_cpu_clock_timers[8];
+static u32 mali_cpu_clock_last_value[8] = {0,};
+#endif
+
+/* Export symbols from common code: mali_user_settings.c */
+#include "mali_user_settings_db.h"
+EXPORT_SYMBOL(mali_set_user_setting);
+EXPORT_SYMBOL(mali_get_user_setting);
+
+static char mali_dev_name[] = "mali"; /* should be const, but the functions we call requires non-cost */
+
+/* This driver only supports one Mali device, and this variable stores this single platform device */
+struct platform_device *mali_platform_device = NULL;
+
+/* This driver only supports one Mali device, and this variable stores the exposed misc device (/dev/mali) */
+static struct miscdevice mali_miscdevice = { 0, };
+
+static int mali_miscdevice_register(struct platform_device *pdev);
+static void mali_miscdevice_unregister(void);
+
+static int mali_open(struct inode *inode, struct file *filp);
+static int mali_release(struct inode *inode, struct file *filp);
+#ifdef HAVE_UNLOCKED_IOCTL
+static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
+#else
+static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg);
+#endif
+
+static int mali_probe(struct platform_device *pdev);
+static int mali_remove(struct platform_device *pdev);
+
+static int mali_driver_suspend_scheduler(struct device *dev);
+static int mali_driver_resume_scheduler(struct device *dev);
+
+#ifdef CONFIG_PM_RUNTIME
+static int mali_driver_runtime_suspend(struct device *dev);
+static int mali_driver_runtime_resume(struct device *dev);
+static int mali_driver_runtime_idle(struct device *dev);
+#endif
+
+#if defined(MALI_FAKE_PLATFORM_DEVICE)
+#if defined(CONFIG_MALI_DT)
+extern int mali_platform_device_init(struct platform_device *device);
+extern int mali_platform_device_deinit(struct platform_device *device);
+#else
+extern int mali_platform_device_register(void);
+extern int mali_platform_device_unregister(void);
+#endif
+#endif
+
+/* Linux power management operations provided by the Mali device driver */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
+struct pm_ext_ops mali_dev_ext_pm_ops = {
+ .base =
+ {
+ .suspend = mali_driver_suspend_scheduler,
+ .resume = mali_driver_resume_scheduler,
+ .freeze = mali_driver_suspend_scheduler,
+ .thaw = mali_driver_resume_scheduler,
+ },
+};
+#else
+static const struct dev_pm_ops mali_dev_pm_ops = {
+#ifdef CONFIG_PM_RUNTIME
+ .runtime_suspend = mali_driver_runtime_suspend,
+ .runtime_resume = mali_driver_runtime_resume,
+ .runtime_idle = mali_driver_runtime_idle,
+#endif
+ .suspend = mali_driver_suspend_scheduler,
+ .resume = mali_driver_resume_scheduler,
+ .freeze = mali_driver_suspend_scheduler,
+ .thaw = mali_driver_resume_scheduler,
+ .poweroff = mali_driver_suspend_scheduler,
+};
+#endif
+
+#ifdef CONFIG_MALI_DT
+static struct of_device_id base_dt_ids[] = {
+ {.compatible = "arm,mali-300"},
+ {.compatible = "arm,mali-400"},
+ {.compatible = "arm,mali-450"},
+ {.compatible = "arm,mali-470"},
+ {},
+};
+
+MODULE_DEVICE_TABLE(of, base_dt_ids);
+#endif
+
+/* The Mali device driver struct */
+static struct platform_driver mali_platform_driver = {
+ .probe = mali_probe,
+ .remove = mali_remove,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
+ .pm = &mali_dev_ext_pm_ops,
+#endif
+ .driver =
+ {
+ .name = MALI_GPU_NAME_UTGARD,
+ .owner = THIS_MODULE,
+ .bus = &platform_bus_type,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+ .pm = &mali_dev_pm_ops,
+#endif
+#ifdef CONFIG_MALI_DT
+ .of_match_table = of_match_ptr(base_dt_ids),
+#endif
+ },
+};
+
+/* Linux misc device operations (/dev/mali) */
+struct file_operations mali_fops = {
+ .owner = THIS_MODULE,
+ .open = mali_open,
+ .release = mali_release,
+#ifdef HAVE_UNLOCKED_IOCTL
+ .unlocked_ioctl = mali_ioctl,
+#else
+ .ioctl = mali_ioctl,
+#endif
+ .compat_ioctl = mali_ioctl,
+ .mmap = mali_mmap
+};
+
+#if MALI_ENABLE_CPU_CYCLES
+void mali_init_cpu_time_counters(int reset, int enable_divide_by_64)
+{
+ /* The CPU assembly reference used is: ARM Architecture Reference Manual ARMv7-AR C.b */
+ u32 write_value;
+
+ /* See B4.1.116 PMCNTENSET, Performance Monitors Count Enable Set register, VMSA */
+ /* setting p15 c9 c12 1 to 0x8000000f==CPU_CYCLE_ENABLE |EVENT_3_ENABLE|EVENT_2_ENABLE|EVENT_1_ENABLE|EVENT_0_ENABLE */
+ asm volatile("mcr p15, 0, %0, c9, c12, 1" :: "r"(0x8000000f));
+
+
+ /* See B4.1.117 PMCR, Performance Monitors Control Register. Writing to p15, c9, c12, 0 */
+ write_value = 1 << 0; /* Bit 0 set. Enable counters */
+ if (reset) {
+ write_value |= 1 << 1; /* Reset event counters */
+ write_value |= 1 << 2; /* Reset cycle counter */
+ }
+ if (enable_divide_by_64) {
+ write_value |= 1 << 3; /* Enable the Clock divider by 64 */
+ }
+ write_value |= 1 << 4; /* Export enable. Not needed */
+ asm volatile("MCR p15, 0, %0, c9, c12, 0\t\n" :: "r"(write_value));
+
+ /* PMOVSR Overflow Flag Status Register - Clear Clock and Event overflows */
+ asm volatile("MCR p15, 0, %0, c9, c12, 3\t\n" :: "r"(0x8000000f));
+
+
+ /* See B4.1.124 PMUSERENR - setting p15 c9 c14 to 1" */
+ /* User mode access to the Performance Monitors enabled. */
+ /* Lets User space read cpu clock cycles */
+ asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(1));
+}
+
+/** A timer function that configures the cycle clock counter on current CPU.
+ * The function \a mali_init_cpu_time_counters_on_all_cpus sets up this
+ * function to trigger on all Cpus during module load.
+ */
+static void mali_init_cpu_clock_timer_func(unsigned long data)
+{
+ int reset_counters, enable_divide_clock_counter_by_64;
+ int current_cpu = raw_smp_processor_id();
+ unsigned int sample0;
+ unsigned int sample1;
+
+ MALI_IGNORE(data);
+
+ reset_counters = 1;
+ enable_divide_clock_counter_by_64 = 0;
+ mali_init_cpu_time_counters(reset_counters, enable_divide_clock_counter_by_64);
+
+ sample0 = mali_get_cpu_cyclecount();
+ sample1 = mali_get_cpu_cyclecount();
+
+ MALI_DEBUG_PRINT(3, ("Init Cpu %d cycle counter- First two samples: %08x %08x \n", current_cpu, sample0, sample1));
+}
+
+/** A timer functions for storing current time on all cpus.
+ * Used for checking if the clocks have similar values or if they are drifting.
+ */
+static void mali_print_cpu_clock_timer_func(unsigned long data)
+{
+ int current_cpu = raw_smp_processor_id();
+ unsigned int sample0;
+
+ MALI_IGNORE(data);
+ sample0 = mali_get_cpu_cyclecount();
+ if (current_cpu < 8) {
+ mali_cpu_clock_last_value[current_cpu] = sample0;
+ }
+}
+
+/** Init the performance registers on all CPUs to count clock cycles.
+ * For init \a print_only should be 0.
+ * If \a print_only is 1, it will intead print the current clock value of all CPUs.
+ */
+void mali_init_cpu_time_counters_on_all_cpus(int print_only)
+{
+ int i = 0;
+ int cpu_number;
+ int jiffies_trigger;
+ int jiffies_wait;
+
+ jiffies_wait = 2;
+ jiffies_trigger = jiffies + jiffies_wait;
+
+ for (i = 0 ; i < 8 ; i++) {
+ init_timer(&mali_init_cpu_clock_timers[i]);
+ if (print_only) mali_init_cpu_clock_timers[i].function = mali_print_cpu_clock_timer_func;
+ else mali_init_cpu_clock_timers[i].function = mali_init_cpu_clock_timer_func;
+ mali_init_cpu_clock_timers[i].expires = jiffies_trigger ;
+ }
+ cpu_number = cpumask_first(cpu_online_mask);
+ for (i = 0 ; i < 8 ; i++) {
+ int next_cpu;
+ add_timer_on(&mali_init_cpu_clock_timers[i], cpu_number);
+ next_cpu = cpumask_next(cpu_number, cpu_online_mask);
+ if (next_cpu >= nr_cpu_ids) break;
+ cpu_number = next_cpu;
+ }
+
+ while (jiffies_wait) jiffies_wait = schedule_timeout_uninterruptible(jiffies_wait);
+
+ for (i = 0 ; i < 8 ; i++) {
+ del_timer_sync(&mali_init_cpu_clock_timers[i]);
+ }
+
+ if (print_only) {
+ if ((0 == mali_cpu_clock_last_value[2]) && (0 == mali_cpu_clock_last_value[3])) {
+ /* Diff can be printed if we want to check if the clocks are in sync
+ int diff = mali_cpu_clock_last_value[0] - mali_cpu_clock_last_value[1];*/
+ MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1]));
+ } else {
+ MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1], mali_cpu_clock_last_value[2], mali_cpu_clock_last_value[3]));
+ }
+ }
+}
+#endif
+
+int mali_module_init(void)
+{
+ int err = 0;
+
+ MALI_DEBUG_PRINT(2, ("Inserting Mali v%d device driver. \n", _MALI_API_VERSION));
+ MALI_DEBUG_PRINT(2, ("Compiled: %s, time: %s.\n", __DATE__, __TIME__));
+ MALI_DEBUG_PRINT(2, ("Driver revision: %s\n", SVN_REV_STRING));
+
+#if MALI_ENABLE_CPU_CYCLES
+ mali_init_cpu_time_counters_on_all_cpus(0);
+ MALI_DEBUG_PRINT(2, ("CPU cycle counter setup complete\n"));
+ /* Printing the current cpu counters */
+ mali_init_cpu_time_counters_on_all_cpus(1);
+#endif
+
+ /* Initialize module wide settings */
+#ifdef MALI_FAKE_PLATFORM_DEVICE
+#ifndef CONFIG_MALI_DT
+ MALI_DEBUG_PRINT(2, ("mali_module_init() registering device\n"));
+ err = mali_platform_device_register();
+ if (0 != err) {
+ return err;
+ }
+#endif
+#endif
+
+ MALI_DEBUG_PRINT(2, ("mali_module_init() registering driver\n"));
+
+ err = platform_driver_register(&mali_platform_driver);
+
+ if (0 != err) {
+ MALI_DEBUG_PRINT(2, ("mali_module_init() Failed to register driver (%d)\n", err));
+#ifdef MALI_FAKE_PLATFORM_DEVICE
+#ifndef CONFIG_MALI_DT
+ mali_platform_device_unregister();
+#endif
+#endif
+ mali_platform_device = NULL;
+ return err;
+ }
+
+#if defined(CONFIG_MALI400_INTERNAL_PROFILING)
+ err = _mali_internal_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE);
+ if (0 != err) {
+ /* No biggie if we wheren't able to initialize the profiling */
+ MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n"));
+ }
+#endif
+
+ /* Tracing the current frequency and voltage from boot/insmod*/
+#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
+ /* Just call mali_get_current_gpu_clk_item(),to record current clk info.*/
+ mali_get_current_gpu_clk_item(&mali_gpu_clk[0]);
+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
+ MALI_PROFILING_EVENT_CHANNEL_GPU |
+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
+ mali_gpu_clk[0].clock,
+ mali_gpu_clk[0].vol / 1000,
+ 0, 0, 0);
+#endif
+
+ MALI_PRINT(("Mali device driver loaded\n"));
+
+ return 0; /* Success */
+}
+
+void mali_module_exit(void)
+{
+ MALI_DEBUG_PRINT(2, ("Unloading Mali v%d device driver.\n", _MALI_API_VERSION));
+
+ MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering driver\n"));
+
+ platform_driver_unregister(&mali_platform_driver);
+
+#if defined(MALI_FAKE_PLATFORM_DEVICE)
+#ifndef CONFIG_MALI_DT
+ MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering device\n"));
+ mali_platform_device_unregister();
+#endif
+#endif
+
+ /* Tracing the current frequency and voltage from rmmod*/
+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
+ MALI_PROFILING_EVENT_CHANNEL_GPU |
+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
+ 0,
+ 0,
+ 0, 0, 0);
+
+#if defined(CONFIG_MALI400_INTERNAL_PROFILING)
+ _mali_internal_profiling_term();
+#endif
+
+ MALI_PRINT(("Mali device driver unloaded\n"));
+}
+
+static int mali_probe(struct platform_device *pdev)
+{
+ int err;
+
+ MALI_DEBUG_PRINT(2, ("mali_probe(): Called for platform device %s\n", pdev->name));
+
+ if (NULL != mali_platform_device) {
+ /* Already connected to a device, return error */
+ MALI_PRINT_ERROR(("mali_probe(): The Mali driver is already connected with a Mali device."));
+ return -EEXIST;
+ }
+
+ mali_platform_device = pdev;
+
+#ifdef CONFIG_MALI_DT
+ /* If we use DT to initialize our DDK, we have to prepare somethings. */
+ err = mali_platform_device_init(mali_platform_device);
+ if (0 != err) {
+ MALI_PRINT_ERROR(("mali_probe(): Failed to initialize platform device."));
+ return -EFAULT;
+ }
+#endif
+
+ if (_MALI_OSK_ERR_OK == _mali_osk_wq_init()) {
+ /* Initialize the Mali GPU HW specified by pdev */
+ if (_MALI_OSK_ERR_OK == mali_initialize_subsystems()) {
+ /* Register a misc device (so we are accessible from user space) */
+ err = mali_miscdevice_register(pdev);
+ if (0 == err) {
+ /* Setup sysfs entries */
+ err = mali_sysfs_register(mali_dev_name);
+
+ if (0 == err) {
+ MALI_DEBUG_PRINT(2, ("mali_probe(): Successfully initialized driver for platform device %s\n", pdev->name));
+
+ return 0;
+ } else {
+ MALI_PRINT_ERROR(("mali_probe(): failed to register sysfs entries"));
+ }
+ mali_miscdevice_unregister();
+ } else {
+ MALI_PRINT_ERROR(("mali_probe(): failed to register Mali misc device."));
+ }
+ mali_terminate_subsystems();
+ } else {
+ MALI_PRINT_ERROR(("mali_probe(): Failed to initialize Mali device driver."));
+ }
+ _mali_osk_wq_term();
+ }
+
+ mali_platform_device = NULL;
+ return -EFAULT;
+}
+
+static int mali_remove(struct platform_device *pdev)
+{
+ MALI_DEBUG_PRINT(2, ("mali_remove() called for platform device %s\n", pdev->name));
+ mali_sysfs_unregister();
+ mali_miscdevice_unregister();
+ mali_terminate_subsystems();
+ _mali_osk_wq_term();
+#ifdef CONFIG_MALI_DT
+ mali_platform_device_deinit(mali_platform_device);
+#endif
+ mali_platform_device = NULL;
+ return 0;
+}
+
+static int mali_miscdevice_register(struct platform_device *pdev)
+{
+ int err;
+
+ mali_miscdevice.minor = MISC_DYNAMIC_MINOR;
+ mali_miscdevice.name = mali_dev_name;
+ mali_miscdevice.fops = &mali_fops;
+ mali_miscdevice.parent = get_device(&pdev->dev);
+
+ err = misc_register(&mali_miscdevice);
+ if (0 != err) {
+ MALI_PRINT_ERROR(("Failed to register misc device, misc_register() returned %d\n", err));
+ }
+
+ return err;
+}
+
+static void mali_miscdevice_unregister(void)
+{
+ misc_deregister(&mali_miscdevice);
+}
+
+static int mali_driver_suspend_scheduler(struct device *dev)
+{
+ mali_pm_os_suspend(MALI_TRUE);
+ /* Tracing the frequency and voltage after mali is suspended */
+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
+ MALI_PROFILING_EVENT_CHANNEL_GPU |
+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
+ 0,
+ 0,
+ 0, 0, 0);
+ return 0;
+}
+
+static int mali_driver_resume_scheduler(struct device *dev)
+{
+ /* Tracing the frequency and voltage after mali is resumed */
+#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
+ /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/
+ if (is_first_resume == 1) {
+ mali_get_current_gpu_clk_item(&mali_gpu_clk[1]);
+ is_first_resume = 0;
+ }
+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
+ MALI_PROFILING_EVENT_CHANNEL_GPU |
+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
+ mali_gpu_clk[1].clock,
+ mali_gpu_clk[1].vol / 1000,
+ 0, 0, 0);
+#endif
+ mali_pm_os_resume();
+ return 0;
+}
+
+#ifdef CONFIG_PM_RUNTIME
+static int mali_driver_runtime_suspend(struct device *dev)
+{
+ if (MALI_TRUE == mali_pm_runtime_suspend()) {
+ /* Tracing the frequency and voltage after mali is suspended */
+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
+ MALI_PROFILING_EVENT_CHANNEL_GPU |
+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
+ 0,
+ 0,
+ 0, 0, 0);
+
+ return 0;
+ } else {
+ return -EBUSY;
+ }
+}
+
+static int mali_driver_runtime_resume(struct device *dev)
+{
+ /* Tracing the frequency and voltage after mali is resumed */
+#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
+ /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/
+ if (is_first_resume == 1) {
+ mali_get_current_gpu_clk_item(&mali_gpu_clk[1]);
+ is_first_resume = 0;
+ }
+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
+ MALI_PROFILING_EVENT_CHANNEL_GPU |
+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
+ mali_gpu_clk[1].clock,
+ mali_gpu_clk[1].vol / 1000,
+ 0, 0, 0);
+#endif
+
+ mali_pm_runtime_resume();
+ return 0;
+}
+
+static int mali_driver_runtime_idle(struct device *dev)
+{
+ /* Nothing to do */
+ return 0;
+}
+#endif
+
+static int mali_open(struct inode *inode, struct file *filp)
+{
+ struct mali_session_data *session_data;
+ _mali_osk_errcode_t err;
+
+ /* input validation */
+ if (mali_miscdevice.minor != iminor(inode)) {
+ MALI_PRINT_ERROR(("mali_open() Minor does not match\n"));
+ return -ENODEV;
+ }
+
+ /* allocated struct to track this session */
+ err = _mali_ukk_open((void **)&session_data);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ /* initialize file pointer */
+ filp->f_pos = 0;
+
+ /* link in our session data */
+ filp->private_data = (void *)session_data;
+
+ filp->f_mapping = mali_mem_swap_get_global_swap_file()->f_mapping;
+
+ return 0;
+}
+
+static int mali_release(struct inode *inode, struct file *filp)
+{
+ _mali_osk_errcode_t err;
+
+ /* input validation */
+ if (mali_miscdevice.minor != iminor(inode)) {
+ MALI_PRINT_ERROR(("mali_release() Minor does not match\n"));
+ return -ENODEV;
+ }
+
+ err = _mali_ukk_close((void **)&filp->private_data);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ return 0;
+}
+
+int map_errcode(_mali_osk_errcode_t err)
+{
+ switch (err) {
+ case _MALI_OSK_ERR_OK :
+ return 0;
+ case _MALI_OSK_ERR_FAULT:
+ return -EFAULT;
+ case _MALI_OSK_ERR_INVALID_FUNC:
+ return -ENOTTY;
+ case _MALI_OSK_ERR_INVALID_ARGS:
+ return -EINVAL;
+ case _MALI_OSK_ERR_NOMEM:
+ return -ENOMEM;
+ case _MALI_OSK_ERR_TIMEOUT:
+ return -ETIMEDOUT;
+ case _MALI_OSK_ERR_RESTARTSYSCALL:
+ return -ERESTARTSYS;
+ case _MALI_OSK_ERR_ITEM_NOT_FOUND:
+ return -ENOENT;
+ default:
+ return -EFAULT;
+ }
+}
+
+#ifdef HAVE_UNLOCKED_IOCTL
+static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+#else
+static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
+#endif
+{
+ int err;
+ struct mali_session_data *session_data;
+
+#ifndef HAVE_UNLOCKED_IOCTL
+ /* inode not used */
+ (void)inode;
+#endif
+
+ MALI_DEBUG_PRINT(7, ("Ioctl received 0x%08X 0x%08lX\n", cmd, arg));
+
+ session_data = (struct mali_session_data *)filp->private_data;
+ if (NULL == session_data) {
+ MALI_DEBUG_PRINT(7, ("filp->private_data was NULL\n"));
+ return -ENOTTY;
+ }
+
+ if (NULL == (void *)arg) {
+ MALI_DEBUG_PRINT(7, ("arg was NULL\n"));
+ return -ENOTTY;
+ }
+
+ switch (cmd) {
+ case MALI_IOC_WAIT_FOR_NOTIFICATION:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_wait_for_notification_s), sizeof(u64)));
+ err = wait_for_notification_wrapper(session_data, (_mali_uk_wait_for_notification_s __user *)arg);
+ break;
+
+ case MALI_IOC_GET_API_VERSION_V2:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_api_version_v2_s), sizeof(u64)));
+ err = get_api_version_v2_wrapper(session_data, (_mali_uk_get_api_version_v2_s __user *)arg);
+ break;
+
+ case MALI_IOC_GET_API_VERSION:
+ err = get_api_version_wrapper(session_data, (_mali_uk_get_api_version_s __user *)arg);
+ break;
+
+ case MALI_IOC_POST_NOTIFICATION:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_post_notification_s), sizeof(u64)));
+ err = post_notification_wrapper(session_data, (_mali_uk_post_notification_s __user *)arg);
+ break;
+
+ case MALI_IOC_GET_USER_SETTINGS:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_user_settings_s), sizeof(u64)));
+ err = get_user_settings_wrapper(session_data, (_mali_uk_get_user_settings_s __user *)arg);
+ break;
+
+ case MALI_IOC_REQUEST_HIGH_PRIORITY:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_request_high_priority_s), sizeof(u64)));
+ err = request_high_priority_wrapper(session_data, (_mali_uk_request_high_priority_s __user *)arg);
+ break;
+
+ case MALI_IOC_PENDING_SUBMIT:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pending_submit_s), sizeof(u64)));
+ err = pending_submit_wrapper(session_data, (_mali_uk_pending_submit_s __user *)arg);
+ break;
+
+#if defined(CONFIG_MALI400_PROFILING)
+ case MALI_IOC_PROFILING_ADD_EVENT:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_add_event_s), sizeof(u64)));
+ err = profiling_add_event_wrapper(session_data, (_mali_uk_profiling_add_event_s __user *)arg);
+ break;
+
+ case MALI_IOC_PROFILING_REPORT_SW_COUNTERS:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_sw_counters_report_s), sizeof(u64)));
+ err = profiling_report_sw_counters_wrapper(session_data, (_mali_uk_sw_counters_report_s __user *)arg);
+ break;
+
+ case MALI_IOC_PROFILING_STREAM_FD_GET:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_stream_fd_get_s), sizeof(u64)));
+ err = profiling_get_stream_fd_wrapper(session_data, (_mali_uk_profiling_stream_fd_get_s __user *)arg);
+ break;
+
+ case MALI_IOC_PROILING_CONTROL_SET:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_control_set_s), sizeof(u64)));
+ err = profiling_control_set_wrapper(session_data, (_mali_uk_profiling_control_set_s __user *)arg);
+ break;
+#else
+
+ case MALI_IOC_PROFILING_ADD_EVENT: /* FALL-THROUGH */
+ case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: /* FALL-THROUGH */
+ MALI_DEBUG_PRINT(2, ("Profiling not supported\n"));
+ err = -ENOTTY;
+ break;
+#endif
+
+ case MALI_IOC_PROFILING_MEMORY_USAGE_GET:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64)));
+ err = mem_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_ALLOC:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_alloc_mem_s), sizeof(u64)));
+ err = mem_alloc_wrapper(session_data, (_mali_uk_alloc_mem_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_FREE:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_free_mem_s), sizeof(u64)));
+ err = mem_free_wrapper(session_data, (_mali_uk_free_mem_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_BIND:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_bind_mem_s), sizeof(u64)));
+ err = mem_bind_wrapper(session_data, (_mali_uk_bind_mem_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_UNBIND:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unbind_mem_s), sizeof(u64)));
+ err = mem_unbind_wrapper(session_data, (_mali_uk_unbind_mem_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_COW:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_mem_s), sizeof(u64)));
+ err = mem_cow_wrapper(session_data, (_mali_uk_cow_mem_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_COW_MODIFY_RANGE:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_modify_range_s), sizeof(u64)));
+ err = mem_cow_modify_range_wrapper(session_data, (_mali_uk_cow_modify_range_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_RESIZE:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_resize_s), sizeof(u64)));
+ err = mem_resize_mem_wrapper(session_data, (_mali_uk_mem_resize_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_WRITE_SAFE:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64)));
+ err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64)));
+ err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64)));
+ err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg);
+ break;
+
+ case MALI_IOC_MEM_DMA_BUF_GET_SIZE:
+#ifdef CONFIG_DMA_SHARED_BUFFER
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dma_buf_get_size_s), sizeof(u64)));
+ err = mali_dma_buf_get_size(session_data, (_mali_uk_dma_buf_get_size_s __user *)arg);
+#else
+ MALI_DEBUG_PRINT(2, ("DMA-BUF not supported\n"));
+ err = -ENOTTY;
+#endif
+ break;
+
+ case MALI_IOC_PP_START_JOB:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_start_job_s), sizeof(u64)));
+ err = pp_start_job_wrapper(session_data, (_mali_uk_pp_start_job_s __user *)arg);
+ break;
+
+ case MALI_IOC_PP_AND_GP_START_JOB:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_and_gp_start_job_s), sizeof(u64)));
+ err = pp_and_gp_start_job_wrapper(session_data, (_mali_uk_pp_and_gp_start_job_s __user *)arg);
+ break;
+
+ case MALI_IOC_PP_NUMBER_OF_CORES_GET:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_number_of_cores_s), sizeof(u64)));
+ err = pp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_pp_number_of_cores_s __user *)arg);
+ break;
+
+ case MALI_IOC_PP_CORE_VERSION_GET:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_core_version_s), sizeof(u64)));
+ err = pp_get_core_version_wrapper(session_data, (_mali_uk_get_pp_core_version_s __user *)arg);
+ break;
+
+ case MALI_IOC_PP_DISABLE_WB:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_disable_wb_s), sizeof(u64)));
+ err = pp_disable_wb_wrapper(session_data, (_mali_uk_pp_disable_wb_s __user *)arg);
+ break;
+
+ case MALI_IOC_GP2_START_JOB:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_start_job_s), sizeof(u64)));
+ err = gp_start_job_wrapper(session_data, (_mali_uk_gp_start_job_s __user *)arg);
+ break;
+
+ case MALI_IOC_GP2_NUMBER_OF_CORES_GET:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_number_of_cores_s), sizeof(u64)));
+ err = gp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_gp_number_of_cores_s __user *)arg);
+ break;
+
+ case MALI_IOC_GP2_CORE_VERSION_GET:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_core_version_s), sizeof(u64)));
+ err = gp_get_core_version_wrapper(session_data, (_mali_uk_get_gp_core_version_s __user *)arg);
+ break;
+
+ case MALI_IOC_GP2_SUSPEND_RESPONSE:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_suspend_response_s), sizeof(u64)));
+ err = gp_suspend_response_wrapper(session_data, (_mali_uk_gp_suspend_response_s __user *)arg);
+ break;
+
+ case MALI_IOC_VSYNC_EVENT_REPORT:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_vsync_event_report_s), sizeof(u64)));
+ err = vsync_event_report_wrapper(session_data, (_mali_uk_vsync_event_report_s __user *)arg);
+ break;
+
+ case MALI_IOC_TIMELINE_GET_LATEST_POINT:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_get_latest_point_s), sizeof(u64)));
+ err = timeline_get_latest_point_wrapper(session_data, (_mali_uk_timeline_get_latest_point_s __user *)arg);
+ break;
+ case MALI_IOC_TIMELINE_WAIT:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_wait_s), sizeof(u64)));
+ err = timeline_wait_wrapper(session_data, (_mali_uk_timeline_wait_s __user *)arg);
+ break;
+ case MALI_IOC_TIMELINE_CREATE_SYNC_FENCE:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_create_sync_fence_s), sizeof(u64)));
+ err = timeline_create_sync_fence_wrapper(session_data, (_mali_uk_timeline_create_sync_fence_s __user *)arg);
+ break;
+ case MALI_IOC_SOFT_JOB_START:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_start_s), sizeof(u64)));
+ err = soft_job_start_wrapper(session_data, (_mali_uk_soft_job_start_s __user *)arg);
+ break;
+ case MALI_IOC_SOFT_JOB_SIGNAL:
+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_signal_s), sizeof(u64)));
+ err = soft_job_signal_wrapper(session_data, (_mali_uk_soft_job_signal_s __user *)arg);
+ break;
+
+ default:
+ MALI_DEBUG_PRINT(2, ("No handler for ioctl 0x%08X 0x%08lX\n", cmd, arg));
+ err = -ENOTTY;
+ };
+
+ return err;
+}
+
+
+module_init(mali_module_init);
+module_exit(mali_module_exit);
+
+MODULE_LICENSE(MALI_KERNEL_LINUX_LICENSE);
+MODULE_AUTHOR("ARM Ltd.");
+MODULE_VERSION(SVN_REV_STRING);
diff --git a/drivers/gpu/arm/utgard/linux/mali_kernel_linux.h b/drivers/gpu/arm/utgard/linux/mali_kernel_linux.h
new file mode 100644
index 000000000000..f6042712d1e3
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_kernel_linux.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_KERNEL_LINUX_H__
+#define __MALI_KERNEL_LINUX_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <linux/cdev.h> /* character device definitions */
+#include <linux/idr.h>
+#include <linux/rbtree.h>
+#include "mali_kernel_license.h"
+#include "mali_osk_types.h"
+
+extern struct platform_device *mali_platform_device;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __MALI_KERNEL_LINUX_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.c b/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.c
new file mode 100644
index 000000000000..4f06ca14bf48
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.c
@@ -0,0 +1,1410 @@
+/**
+ * Copyright (C) 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+
+/**
+ * @file mali_kernel_sysfs.c
+ * Implementation of some sysfs data exports
+ */
+
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include "mali_kernel_license.h"
+#include "mali_kernel_common.h"
+#include "mali_ukk.h"
+
+#if MALI_LICENSE_IS_GPL
+
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+#include <asm/uaccess.h>
+#include <linux/module.h>
+#include <linux/mali/mali_utgard.h>
+#include "mali_kernel_sysfs.h"
+#if defined(CONFIG_MALI400_INTERNAL_PROFILING)
+#include <linux/slab.h>
+#include "mali_osk_profiling.h"
+#endif
+
+#include <linux/mali/mali_utgard.h>
+#include "mali_pm.h"
+#include "mali_pmu.h"
+#include "mali_group.h"
+#include "mali_gp.h"
+#include "mali_pp.h"
+#include "mali_l2_cache.h"
+#include "mali_hw_core.h"
+#include "mali_kernel_core.h"
+#include "mali_user_settings_db.h"
+#include "mali_profiling_internal.h"
+#include "mali_gp_job.h"
+#include "mali_pp_job.h"
+#include "mali_executor.h"
+
+#define PRIVATE_DATA_COUNTER_MAKE_GP(src) (src)
+#define PRIVATE_DATA_COUNTER_MAKE_PP(src) ((1 << 24) | src)
+#define PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(src, sub_job) ((1 << 24) | (1 << 16) | (sub_job << 8) | src)
+#define PRIVATE_DATA_COUNTER_IS_PP(a) ((((a) >> 24) & 0xFF) ? MALI_TRUE : MALI_FALSE)
+#define PRIVATE_DATA_COUNTER_GET_SRC(a) (a & 0xFF)
+#define PRIVATE_DATA_COUNTER_IS_SUB_JOB(a) ((((a) >> 16) & 0xFF) ? MALI_TRUE : MALI_FALSE)
+#define PRIVATE_DATA_COUNTER_GET_SUB_JOB(a) (((a) >> 8) & 0xFF)
+
+#define POWER_BUFFER_SIZE 3
+
+static struct dentry *mali_debugfs_dir = NULL;
+
+typedef enum {
+ _MALI_DEVICE_SUSPEND,
+ _MALI_DEVICE_RESUME,
+ _MALI_DEVICE_DVFS_PAUSE,
+ _MALI_DEVICE_DVFS_RESUME,
+ _MALI_MAX_EVENTS
+} _mali_device_debug_power_events;
+
+static const char *const mali_power_events[_MALI_MAX_EVENTS] = {
+ [_MALI_DEVICE_SUSPEND] = "suspend",
+ [_MALI_DEVICE_RESUME] = "resume",
+ [_MALI_DEVICE_DVFS_PAUSE] = "dvfs_pause",
+ [_MALI_DEVICE_DVFS_RESUME] = "dvfs_resume",
+};
+
+static mali_bool power_always_on_enabled = MALI_FALSE;
+
+static int open_copy_private_data(struct inode *inode, struct file *filp)
+{
+ filp->private_data = inode->i_private;
+ return 0;
+}
+
+static ssize_t group_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp)
+{
+ int r;
+ char buffer[64];
+ struct mali_group *group;
+
+ group = (struct mali_group *)filp->private_data;
+ MALI_DEBUG_ASSERT_POINTER(group);
+
+ r = snprintf(buffer, 64, "%u\n",
+ mali_executor_group_is_disabled(group) ? 0 : 1);
+
+ return simple_read_from_buffer(buf, count, offp, buffer, r);
+}
+
+static ssize_t group_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp)
+{
+ int r;
+ char buffer[64];
+ unsigned long val;
+ struct mali_group *group;
+
+ group = (struct mali_group *)filp->private_data;
+ MALI_DEBUG_ASSERT_POINTER(group);
+
+ if (count >= sizeof(buffer)) {
+ return -ENOMEM;
+ }
+
+ if (copy_from_user(&buffer[0], buf, count)) {
+ return -EFAULT;
+ }
+ buffer[count] = '\0';
+
+ r = kstrtoul(&buffer[0], 10, &val);
+ if (0 != r) {
+ return -EINVAL;
+ }
+
+ switch (val) {
+ case 1:
+ mali_executor_group_enable(group);
+ break;
+ case 0:
+ mali_executor_group_disable(group);
+ break;
+ default:
+ return -EINVAL;
+ break;
+ }
+
+ *offp += count;
+ return count;
+}
+
+static const struct file_operations group_enabled_fops = {
+ .owner = THIS_MODULE,
+ .open = open_copy_private_data,
+ .read = group_enabled_read,
+ .write = group_enabled_write,
+};
+
+static ssize_t hw_core_base_addr_read(struct file *filp, char __user *buf, size_t count, loff_t *offp)
+{
+ int r;
+ char buffer[64];
+ struct mali_hw_core *hw_core;
+
+ hw_core = (struct mali_hw_core *)filp->private_data;
+ MALI_DEBUG_ASSERT_POINTER(hw_core);
+
+ r = snprintf(buffer, 64, "0x%lX\n", hw_core->phys_addr);
+
+ return simple_read_from_buffer(buf, count, offp, buffer, r);
+}
+
+static const struct file_operations hw_core_base_addr_fops = {
+ .owner = THIS_MODULE,
+ .open = open_copy_private_data,
+ .read = hw_core_base_addr_read,
+};
+
+static ssize_t profiling_counter_src_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ u32 is_pp = PRIVATE_DATA_COUNTER_IS_PP((uintptr_t)filp->private_data);
+ u32 src_id = PRIVATE_DATA_COUNTER_GET_SRC((uintptr_t)filp->private_data);
+ mali_bool is_sub_job = PRIVATE_DATA_COUNTER_IS_SUB_JOB((uintptr_t)filp->private_data);
+ u32 sub_job = PRIVATE_DATA_COUNTER_GET_SUB_JOB((uintptr_t)filp->private_data);
+ char buf[64];
+ int r;
+ u32 val;
+
+ if (MALI_TRUE == is_pp) {
+ /* PP counter */
+ if (MALI_TRUE == is_sub_job) {
+ /* Get counter for a particular sub job */
+ if (0 == src_id) {
+ val = mali_pp_job_get_pp_counter_sub_job_src0(sub_job);
+ } else {
+ val = mali_pp_job_get_pp_counter_sub_job_src1(sub_job);
+ }
+ } else {
+ /* Get default counter for all PP sub jobs */
+ if (0 == src_id) {
+ val = mali_pp_job_get_pp_counter_global_src0();
+ } else {
+ val = mali_pp_job_get_pp_counter_global_src1();
+ }
+ }
+ } else {
+ /* GP counter */
+ if (0 == src_id) {
+ val = mali_gp_job_get_gp_counter_src0();
+ } else {
+ val = mali_gp_job_get_gp_counter_src1();
+ }
+ }
+
+ if (MALI_HW_CORE_NO_COUNTER == val) {
+ r = snprintf(buf, 64, "-1\n");
+ } else {
+ r = snprintf(buf, 64, "%u\n", val);
+ }
+
+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t profiling_counter_src_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ u32 is_pp = PRIVATE_DATA_COUNTER_IS_PP((uintptr_t)filp->private_data);
+ u32 src_id = PRIVATE_DATA_COUNTER_GET_SRC((uintptr_t)filp->private_data);
+ mali_bool is_sub_job = PRIVATE_DATA_COUNTER_IS_SUB_JOB((uintptr_t)filp->private_data);
+ u32 sub_job = PRIVATE_DATA_COUNTER_GET_SUB_JOB((uintptr_t)filp->private_data);
+ char buf[64];
+ long val;
+ int ret;
+
+ if (cnt >= sizeof(buf)) {
+ return -EINVAL;
+ }
+
+ if (copy_from_user(&buf, ubuf, cnt)) {
+ return -EFAULT;
+ }
+
+ buf[cnt] = 0;
+
+ ret = kstrtol(buf, 10, &val);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (val < 0) {
+ /* any negative input will disable counter */
+ val = MALI_HW_CORE_NO_COUNTER;
+ }
+
+ if (MALI_TRUE == is_pp) {
+ /* PP counter */
+ if (MALI_TRUE == is_sub_job) {
+ /* Set counter for a particular sub job */
+ if (0 == src_id) {
+ mali_pp_job_set_pp_counter_sub_job_src0(sub_job, (u32)val);
+ } else {
+ mali_pp_job_set_pp_counter_sub_job_src1(sub_job, (u32)val);
+ }
+ } else {
+ /* Set default counter for all PP sub jobs */
+ if (0 == src_id) {
+ mali_pp_job_set_pp_counter_global_src0((u32)val);
+ } else {
+ mali_pp_job_set_pp_counter_global_src1((u32)val);
+ }
+ }
+ } else {
+ /* GP counter */
+ if (0 == src_id) {
+ mali_gp_job_set_gp_counter_src0((u32)val);
+ } else {
+ mali_gp_job_set_gp_counter_src1((u32)val);
+ }
+ }
+
+ *ppos += cnt;
+ return cnt;
+}
+
+static const struct file_operations profiling_counter_src_fops = {
+ .owner = THIS_MODULE,
+ .open = open_copy_private_data,
+ .read = profiling_counter_src_read,
+ .write = profiling_counter_src_write,
+};
+
+static ssize_t l2_l2x_counter_srcx_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id)
+{
+ char buf[64];
+ int r;
+ u32 val;
+ struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data;
+
+ if (0 == src_id) {
+ val = mali_l2_cache_core_get_counter_src0(l2_core);
+ } else {
+ val = mali_l2_cache_core_get_counter_src1(l2_core);
+ }
+
+ if (MALI_HW_CORE_NO_COUNTER == val) {
+ r = snprintf(buf, 64, "-1\n");
+ } else {
+ r = snprintf(buf, 64, "%u\n", val);
+ }
+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t l2_l2x_counter_srcx_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id)
+{
+ struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data;
+ char buf[64];
+ long val;
+ int ret;
+
+ if (cnt >= sizeof(buf)) {
+ return -EINVAL;
+ }
+
+ if (copy_from_user(&buf, ubuf, cnt)) {
+ return -EFAULT;
+ }
+
+ buf[cnt] = 0;
+
+ ret = kstrtol(buf, 10, &val);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (val < 0) {
+ /* any negative input will disable counter */
+ val = MALI_HW_CORE_NO_COUNTER;
+ }
+
+ mali_l2_cache_core_set_counter_src(l2_core, src_id, (u32)val);
+
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t l2_all_counter_srcx_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id)
+{
+ char buf[64];
+ long val;
+ int ret;
+ u32 l2_id;
+ struct mali_l2_cache_core *l2_cache;
+
+ if (cnt >= sizeof(buf)) {
+ return -EINVAL;
+ }
+
+ if (copy_from_user(&buf, ubuf, cnt)) {
+ return -EFAULT;
+ }
+
+ buf[cnt] = 0;
+
+ ret = kstrtol(buf, 10, &val);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (val < 0) {
+ /* any negative input will disable counter */
+ val = MALI_HW_CORE_NO_COUNTER;
+ }
+
+ l2_id = 0;
+ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id);
+ while (NULL != l2_cache) {
+ mali_l2_cache_core_set_counter_src(l2_cache, src_id, (u32)val);
+
+ /* try next L2 */
+ l2_id++;
+ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id);
+ }
+
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t l2_l2x_counter_src0_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ return l2_l2x_counter_srcx_read(filp, ubuf, cnt, ppos, 0);
+}
+
+static ssize_t l2_l2x_counter_src1_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ return l2_l2x_counter_srcx_read(filp, ubuf, cnt, ppos, 1);
+}
+
+static ssize_t l2_l2x_counter_src0_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ return l2_l2x_counter_srcx_write(filp, ubuf, cnt, ppos, 0);
+}
+
+static ssize_t l2_l2x_counter_src1_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ return l2_l2x_counter_srcx_write(filp, ubuf, cnt, ppos, 1);
+}
+
+static ssize_t l2_all_counter_src0_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ return l2_all_counter_srcx_write(filp, ubuf, cnt, ppos, 0);
+}
+
+static ssize_t l2_all_counter_src1_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ return l2_all_counter_srcx_write(filp, ubuf, cnt, ppos, 1);
+}
+
+static const struct file_operations l2_l2x_counter_src0_fops = {
+ .owner = THIS_MODULE,
+ .open = open_copy_private_data,
+ .read = l2_l2x_counter_src0_read,
+ .write = l2_l2x_counter_src0_write,
+};
+
+static const struct file_operations l2_l2x_counter_src1_fops = {
+ .owner = THIS_MODULE,
+ .open = open_copy_private_data,
+ .read = l2_l2x_counter_src1_read,
+ .write = l2_l2x_counter_src1_write,
+};
+
+static const struct file_operations l2_all_counter_src0_fops = {
+ .owner = THIS_MODULE,
+ .write = l2_all_counter_src0_write,
+};
+
+static const struct file_operations l2_all_counter_src1_fops = {
+ .owner = THIS_MODULE,
+ .write = l2_all_counter_src1_write,
+};
+
+static ssize_t l2_l2x_counter_valx_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id)
+{
+ char buf[64];
+ int r;
+ u32 src0 = 0;
+ u32 val0 = 0;
+ u32 src1 = 0;
+ u32 val1 = 0;
+ u32 val = -1;
+ struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data;
+
+ mali_l2_cache_core_get_counter_values(l2_core, &src0, &val0, &src1, &val1);
+
+ if (0 == src_id) {
+ if (MALI_HW_CORE_NO_COUNTER != val0) {
+ val = val0;
+ }
+ } else {
+ if (MALI_HW_CORE_NO_COUNTER != val1) {
+ val = val1;
+ }
+ }
+
+ r = snprintf(buf, 64, "%u\n", val);
+
+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t l2_l2x_counter_val0_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ return l2_l2x_counter_valx_read(filp, ubuf, cnt, ppos, 0);
+}
+
+static ssize_t l2_l2x_counter_val1_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ return l2_l2x_counter_valx_read(filp, ubuf, cnt, ppos, 1);
+}
+
+static const struct file_operations l2_l2x_counter_val0_fops = {
+ .owner = THIS_MODULE,
+ .open = open_copy_private_data,
+ .read = l2_l2x_counter_val0_read,
+};
+
+static const struct file_operations l2_l2x_counter_val1_fops = {
+ .owner = THIS_MODULE,
+ .open = open_copy_private_data,
+ .read = l2_l2x_counter_val1_read,
+};
+
+static ssize_t power_always_on_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ unsigned long val;
+ int ret;
+ char buf[32];
+
+ cnt = min(cnt, sizeof(buf) - 1);
+ if (copy_from_user(buf, ubuf, cnt)) {
+ return -EFAULT;
+ }
+ buf[cnt] = '\0';
+
+ ret = kstrtoul(buf, 10, &val);
+ if (0 != ret) {
+ return ret;
+ }
+
+ /* Update setting (not exactly thread safe) */
+ if (1 == val && MALI_FALSE == power_always_on_enabled) {
+ power_always_on_enabled = MALI_TRUE;
+ _mali_osk_pm_dev_ref_get_sync();
+ } else if (0 == val && MALI_TRUE == power_always_on_enabled) {
+ power_always_on_enabled = MALI_FALSE;
+ _mali_osk_pm_dev_ref_put();
+ }
+
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t power_always_on_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ if (MALI_TRUE == power_always_on_enabled) {
+ return simple_read_from_buffer(ubuf, cnt, ppos, "1\n", 2);
+ } else {
+ return simple_read_from_buffer(ubuf, cnt, ppos, "0\n", 2);
+ }
+}
+
+static const struct file_operations power_always_on_fops = {
+ .owner = THIS_MODULE,
+ .read = power_always_on_read,
+ .write = power_always_on_write,
+};
+
+static ssize_t power_power_events_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_SUSPEND], strlen(mali_power_events[_MALI_DEVICE_SUSPEND]) - 1)) {
+ mali_pm_os_suspend(MALI_TRUE);
+ } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_RESUME], strlen(mali_power_events[_MALI_DEVICE_RESUME]) - 1)) {
+ mali_pm_os_resume();
+ } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_PAUSE], strlen(mali_power_events[_MALI_DEVICE_DVFS_PAUSE]) - 1)) {
+ mali_dev_pause();
+ } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_RESUME], strlen(mali_power_events[_MALI_DEVICE_DVFS_RESUME]) - 1)) {
+ mali_dev_resume();
+ }
+ *ppos += cnt;
+ return cnt;
+}
+
+static loff_t power_power_events_seek(struct file *file, loff_t offset, int orig)
+{
+ file->f_pos = offset;
+ return 0;
+}
+
+static const struct file_operations power_power_events_fops = {
+ .owner = THIS_MODULE,
+ .write = power_power_events_write,
+ .llseek = power_power_events_seek,
+};
+
+#if MALI_STATE_TRACKING
+static int mali_seq_internal_state_show(struct seq_file *seq_file, void *v)
+{
+ u32 len = 0;
+ u32 size;
+ char *buf;
+
+ size = seq_get_buf(seq_file, &buf);
+
+ if (!size) {
+ return -ENOMEM;
+ }
+
+ /* Create the internal state dump. */
+ len = snprintf(buf + len, size - len, "Mali device driver %s\n", SVN_REV_STRING);
+ len += snprintf(buf + len, size - len, "License: %s\n\n", MALI_KERNEL_LINUX_LICENSE);
+
+ len += _mali_kernel_core_dump_state(buf + len, size - len);
+
+ seq_commit(seq_file, len);
+
+ return 0;
+}
+
+static int mali_seq_internal_state_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, mali_seq_internal_state_show, NULL);
+}
+
+static const struct file_operations mali_seq_internal_state_fops = {
+ .owner = THIS_MODULE,
+ .open = mali_seq_internal_state_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+#endif /* MALI_STATE_TRACKING */
+
+#if defined(CONFIG_MALI400_INTERNAL_PROFILING)
+static ssize_t profiling_record_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ char buf[64];
+ int r;
+
+ r = snprintf(buf, 64, "%u\n", _mali_internal_profiling_is_recording() ? 1 : 0);
+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t profiling_record_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ char buf[64];
+ unsigned long val;
+ int ret;
+
+ if (cnt >= sizeof(buf)) {
+ return -EINVAL;
+ }
+
+ if (copy_from_user(&buf, ubuf, cnt)) {
+ return -EFAULT;
+ }
+
+ buf[cnt] = 0;
+
+ ret = kstrtoul(buf, 10, &val);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (val != 0) {
+ u32 limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; /* This can be made configurable at a later stage if we need to */
+
+ /* check if we are already recording */
+ if (MALI_TRUE == _mali_internal_profiling_is_recording()) {
+ MALI_DEBUG_PRINT(3, ("Recording of profiling events already in progress\n"));
+ return -EFAULT;
+ }
+
+ /* check if we need to clear out an old recording first */
+ if (MALI_TRUE == _mali_internal_profiling_have_recording()) {
+ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_clear()) {
+ MALI_DEBUG_PRINT(3, ("Failed to clear existing recording of profiling events\n"));
+ return -EFAULT;
+ }
+ }
+
+ /* start recording profiling data */
+ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_start(&limit)) {
+ MALI_DEBUG_PRINT(3, ("Failed to start recording of profiling events\n"));
+ return -EFAULT;
+ }
+
+ MALI_DEBUG_PRINT(3, ("Profiling recording started (max %u events)\n", limit));
+ } else {
+ /* stop recording profiling data */
+ u32 count = 0;
+ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_stop(&count)) {
+ MALI_DEBUG_PRINT(2, ("Failed to stop recording of profiling events\n"));
+ return -EFAULT;
+ }
+
+ MALI_DEBUG_PRINT(2, ("Profiling recording stopped (recorded %u events)\n", count));
+ }
+
+ *ppos += cnt;
+ return cnt;
+}
+
+static const struct file_operations profiling_record_fops = {
+ .owner = THIS_MODULE,
+ .read = profiling_record_read,
+ .write = profiling_record_write,
+};
+
+static void *profiling_events_start(struct seq_file *s, loff_t *pos)
+{
+ loff_t *spos;
+
+ /* check if we have data avaiable */
+ if (MALI_TRUE != _mali_internal_profiling_have_recording()) {
+ return NULL;
+ }
+
+ spos = kmalloc(sizeof(loff_t), GFP_KERNEL);
+ if (NULL == spos) {
+ return NULL;
+ }
+
+ *spos = *pos;
+ return spos;
+}
+
+static void *profiling_events_next(struct seq_file *s, void *v, loff_t *pos)
+{
+ loff_t *spos = v;
+
+ /* check if we have data avaiable */
+ if (MALI_TRUE != _mali_internal_profiling_have_recording()) {
+ return NULL;
+ }
+
+ /* check if the next entry actually is avaiable */
+ if (_mali_internal_profiling_get_count() <= (u32)(*spos + 1)) {
+ return NULL;
+ }
+
+ *pos = ++*spos;
+ return spos;
+}
+
+static void profiling_events_stop(struct seq_file *s, void *v)
+{
+ kfree(v);
+}
+
+static int profiling_events_show(struct seq_file *seq_file, void *v)
+{
+ loff_t *spos = v;
+ u32 index;
+ u64 timestamp;
+ u32 event_id;
+ u32 data[5];
+
+ index = (u32) * spos;
+
+ /* Retrieve all events */
+ if (_MALI_OSK_ERR_OK == _mali_internal_profiling_get_event(index, &timestamp, &event_id, data)) {
+ seq_printf(seq_file, "%llu %u %u %u %u %u %u\n", timestamp, event_id, data[0], data[1], data[2], data[3], data[4]);
+ return 0;
+ }
+
+ return 0;
+}
+
+static int profiling_events_show_human_readable(struct seq_file *seq_file, void *v)
+{
+#define MALI_EVENT_ID_IS_HW(event_id) (((event_id & 0x00FF0000) >= MALI_PROFILING_EVENT_CHANNEL_GP0) && ((event_id & 0x00FF0000) <= MALI_PROFILING_EVENT_CHANNEL_PP7))
+
+ static u64 start_time = 0;
+ loff_t *spos = v;
+ u32 index;
+ u64 timestamp;
+ u32 event_id;
+ u32 data[5];
+
+ index = (u32) * spos;
+
+ /* Retrieve all events */
+ if (_MALI_OSK_ERR_OK == _mali_internal_profiling_get_event(index, &timestamp, &event_id, data)) {
+ seq_printf(seq_file, "%llu %u %u %u %u %u %u # ", timestamp, event_id, data[0], data[1], data[2], data[3], data[4]);
+
+ if (0 == index) {
+ start_time = timestamp;
+ }
+
+ seq_printf(seq_file, "[%06u] ", index);
+
+ switch (event_id & 0x0F000000) {
+ case MALI_PROFILING_EVENT_TYPE_SINGLE:
+ seq_printf(seq_file, "SINGLE | ");
+ break;
+ case MALI_PROFILING_EVENT_TYPE_START:
+ seq_printf(seq_file, "START | ");
+ break;
+ case MALI_PROFILING_EVENT_TYPE_STOP:
+ seq_printf(seq_file, "STOP | ");
+ break;
+ case MALI_PROFILING_EVENT_TYPE_SUSPEND:
+ seq_printf(seq_file, "SUSPEND | ");
+ break;
+ case MALI_PROFILING_EVENT_TYPE_RESUME:
+ seq_printf(seq_file, "RESUME | ");
+ break;
+ default:
+ seq_printf(seq_file, "0x%01X | ", (event_id & 0x0F000000) >> 24);
+ break;
+ }
+
+ switch (event_id & 0x00FF0000) {
+ case MALI_PROFILING_EVENT_CHANNEL_SOFTWARE:
+ seq_printf(seq_file, "SW | ");
+ break;
+ case MALI_PROFILING_EVENT_CHANNEL_GP0:
+ seq_printf(seq_file, "GP0 | ");
+ break;
+ case MALI_PROFILING_EVENT_CHANNEL_PP0:
+ seq_printf(seq_file, "PP0 | ");
+ break;
+ case MALI_PROFILING_EVENT_CHANNEL_PP1:
+ seq_printf(seq_file, "PP1 | ");
+ break;
+ case MALI_PROFILING_EVENT_CHANNEL_PP2:
+ seq_printf(seq_file, "PP2 | ");
+ break;
+ case MALI_PROFILING_EVENT_CHANNEL_PP3:
+ seq_printf(seq_file, "PP3 | ");
+ break;
+ case MALI_PROFILING_EVENT_CHANNEL_PP4:
+ seq_printf(seq_file, "PP4 | ");
+ break;
+ case MALI_PROFILING_EVENT_CHANNEL_PP5:
+ seq_printf(seq_file, "PP5 | ");
+ break;
+ case MALI_PROFILING_EVENT_CHANNEL_PP6:
+ seq_printf(seq_file, "PP6 | ");
+ break;
+ case MALI_PROFILING_EVENT_CHANNEL_PP7:
+ seq_printf(seq_file, "PP7 | ");
+ break;
+ case MALI_PROFILING_EVENT_CHANNEL_GPU:
+ seq_printf(seq_file, "GPU | ");
+ break;
+ default:
+ seq_printf(seq_file, "0x%02X | ", (event_id & 0x00FF0000) >> 16);
+ break;
+ }
+
+ if (MALI_EVENT_ID_IS_HW(event_id)) {
+ if (((event_id & 0x0F000000) == MALI_PROFILING_EVENT_TYPE_START) || ((event_id & 0x0F000000) == MALI_PROFILING_EVENT_TYPE_STOP)) {
+ switch (event_id & 0x0000FFFF) {
+ case MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL:
+ seq_printf(seq_file, "PHYSICAL | ");
+ break;
+ case MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL:
+ seq_printf(seq_file, "VIRTUAL | ");
+ break;
+ default:
+ seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF);
+ break;
+ }
+ } else {
+ seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF);
+ }
+ } else {
+ seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF);
+ }
+
+ seq_printf(seq_file, "T0 + 0x%016llX\n", timestamp - start_time);
+
+ return 0;
+ }
+
+ return 0;
+}
+
+static const struct seq_operations profiling_events_seq_ops = {
+ .start = profiling_events_start,
+ .next = profiling_events_next,
+ .stop = profiling_events_stop,
+ .show = profiling_events_show
+};
+
+static int profiling_events_open(struct inode *inode, struct file *file)
+{
+ return seq_open(file, &profiling_events_seq_ops);
+}
+
+static const struct file_operations profiling_events_fops = {
+ .owner = THIS_MODULE,
+ .open = profiling_events_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+
+static const struct seq_operations profiling_events_human_readable_seq_ops = {
+ .start = profiling_events_start,
+ .next = profiling_events_next,
+ .stop = profiling_events_stop,
+ .show = profiling_events_show_human_readable
+};
+
+static int profiling_events_human_readable_open(struct inode *inode, struct file *file)
+{
+ return seq_open(file, &profiling_events_human_readable_seq_ops);
+}
+
+static const struct file_operations profiling_events_human_readable_fops = {
+ .owner = THIS_MODULE,
+ .open = profiling_events_human_readable_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+
+#endif
+
+static int memory_debugfs_show(struct seq_file *s, void *private_data)
+{
+#ifdef MALI_MEM_SWAP_TRACKING
+ seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s %-10s \n"\
+ "=================================================================================================================================\n",
+ "Name (:bytes)", "pid", "mali_mem", "max_mali_mem",
+ "external_mem", "ump_mem", "dma_mem", "swap_mem");
+#else
+ seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s \n"\
+ "========================================================================================================================\n",
+ "Name (:bytes)", "pid", "mali_mem", "max_mali_mem",
+ "external_mem", "ump_mem", "dma_mem");
+#endif
+ mali_session_memory_tracking(s);
+ return 0;
+}
+
+static int memory_debugfs_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, memory_debugfs_show, inode->i_private);
+}
+
+static const struct file_operations memory_usage_fops = {
+ .owner = THIS_MODULE,
+ .open = memory_debugfs_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static ssize_t utilization_gp_pp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ char buf[64];
+ size_t r;
+ u32 uval = _mali_ukk_utilization_gp_pp();
+
+ r = snprintf(buf, 64, "%u\n", uval);
+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t utilization_gp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ char buf[64];
+ size_t r;
+ u32 uval = _mali_ukk_utilization_gp();
+
+ r = snprintf(buf, 64, "%u\n", uval);
+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static ssize_t utilization_pp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ char buf[64];
+ size_t r;
+ u32 uval = _mali_ukk_utilization_pp();
+
+ r = snprintf(buf, 64, "%u\n", uval);
+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+
+static const struct file_operations utilization_gp_pp_fops = {
+ .owner = THIS_MODULE,
+ .read = utilization_gp_pp_read,
+};
+
+static const struct file_operations utilization_gp_fops = {
+ .owner = THIS_MODULE,
+ .read = utilization_gp_read,
+};
+
+static const struct file_operations utilization_pp_fops = {
+ .owner = THIS_MODULE,
+ .read = utilization_pp_read,
+};
+
+static ssize_t user_settings_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ unsigned long val;
+ int ret;
+ _mali_uk_user_setting_t setting;
+ char buf[32];
+
+ cnt = min(cnt, sizeof(buf) - 1);
+ if (copy_from_user(buf, ubuf, cnt)) {
+ return -EFAULT;
+ }
+ buf[cnt] = '\0';
+
+ ret = kstrtoul(buf, 10, &val);
+ if (0 != ret) {
+ return ret;
+ }
+
+ /* Update setting */
+ setting = (_mali_uk_user_setting_t)(filp->private_data);
+ mali_set_user_setting(setting, val);
+
+ *ppos += cnt;
+ return cnt;
+}
+
+static ssize_t user_settings_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+ char buf[64];
+ size_t r;
+ u32 value;
+ _mali_uk_user_setting_t setting;
+
+ setting = (_mali_uk_user_setting_t)(filp->private_data);
+ value = mali_get_user_setting(setting);
+
+ r = snprintf(buf, 64, "%u\n", value);
+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+}
+
+static const struct file_operations user_settings_fops = {
+ .owner = THIS_MODULE,
+ .open = open_copy_private_data,
+ .read = user_settings_read,
+ .write = user_settings_write,
+};
+
+static int mali_sysfs_user_settings_register(void)
+{
+ struct dentry *mali_user_settings_dir = debugfs_create_dir("userspace_settings", mali_debugfs_dir);
+
+ if (mali_user_settings_dir != NULL) {
+ long i;
+ for (i = 0; i < _MALI_UK_USER_SETTING_MAX; i++) {
+ debugfs_create_file(_mali_uk_user_setting_descriptions[i],
+ 0600, mali_user_settings_dir, (void *)i,
+ &user_settings_fops);
+ }
+ }
+
+ return 0;
+}
+
+static ssize_t pp_num_cores_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp)
+{
+ int ret;
+ char buffer[32];
+ unsigned long val;
+
+ if (count >= sizeof(buffer)) {
+ return -ENOMEM;
+ }
+
+ if (copy_from_user(&buffer[0], buf, count)) {
+ return -EFAULT;
+ }
+ buffer[count] = '\0';
+
+ ret = kstrtoul(&buffer[0], 10, &val);
+ if (0 != ret) {
+ return -EINVAL;
+ }
+
+ ret = mali_executor_set_perf_level(val, MALI_TRUE); /* override even if core scaling is disabled */
+ if (ret) {
+ return ret;
+ }
+
+ *offp += count;
+ return count;
+}
+
+static ssize_t pp_num_cores_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp)
+{
+ int r;
+ char buffer[64];
+
+ r = snprintf(buffer, 64, "%u\n", mali_executor_get_num_cores_enabled());
+
+ return simple_read_from_buffer(buf, count, offp, buffer, r);
+}
+
+static const struct file_operations pp_num_cores_enabled_fops = {
+ .owner = THIS_MODULE,
+ .write = pp_num_cores_enabled_write,
+ .read = pp_num_cores_enabled_read,
+ .llseek = default_llseek,
+};
+
+static ssize_t pp_num_cores_total_read(struct file *filp, char __user *buf, size_t count, loff_t *offp)
+{
+ int r;
+ char buffer[64];
+
+ r = snprintf(buffer, 64, "%u\n", mali_executor_get_num_cores_total());
+
+ return simple_read_from_buffer(buf, count, offp, buffer, r);
+}
+
+static const struct file_operations pp_num_cores_total_fops = {
+ .owner = THIS_MODULE,
+ .read = pp_num_cores_total_read,
+};
+
+static ssize_t pp_core_scaling_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp)
+{
+ int ret;
+ char buffer[32];
+ unsigned long val;
+
+ if (count >= sizeof(buffer)) {
+ return -ENOMEM;
+ }
+
+ if (copy_from_user(&buffer[0], buf, count)) {
+ return -EFAULT;
+ }
+ buffer[count] = '\0';
+
+ ret = kstrtoul(&buffer[0], 10, &val);
+ if (0 != ret) {
+ return -EINVAL;
+ }
+
+ switch (val) {
+ case 1:
+ mali_executor_core_scaling_enable();
+ break;
+ case 0:
+ mali_executor_core_scaling_disable();
+ break;
+ default:
+ return -EINVAL;
+ break;
+ }
+
+ *offp += count;
+ return count;
+}
+
+static ssize_t pp_core_scaling_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp)
+{
+ return simple_read_from_buffer(buf, count, offp, mali_executor_core_scaling_is_enabled() ? "1\n" : "0\n", 2);
+}
+static const struct file_operations pp_core_scaling_enabled_fops = {
+ .owner = THIS_MODULE,
+ .write = pp_core_scaling_enabled_write,
+ .read = pp_core_scaling_enabled_read,
+ .llseek = default_llseek,
+};
+
+static ssize_t version_read(struct file *filp, char __user *buf, size_t count, loff_t *offp)
+{
+ int r = 0;
+ char buffer[64];
+
+ switch (mali_kernel_core_get_product_id()) {
+ case _MALI_PRODUCT_ID_MALI200:
+ r = snprintf(buffer, 64, "Mali-200\n");
+ break;
+ case _MALI_PRODUCT_ID_MALI300:
+ r = snprintf(buffer, 64, "Mali-300\n");
+ break;
+ case _MALI_PRODUCT_ID_MALI400:
+ r = snprintf(buffer, 64, "Mali-400 MP\n");
+ break;
+ case _MALI_PRODUCT_ID_MALI450:
+ r = snprintf(buffer, 64, "Mali-450 MP\n");
+ break;
+ case _MALI_PRODUCT_ID_MALI470:
+ r = snprintf(buffer, 64, "Mali-470 MP\n");
+ break;
+ case _MALI_PRODUCT_ID_UNKNOWN:
+ return -EINVAL;
+ break;
+ };
+
+ return simple_read_from_buffer(buf, count, offp, buffer, r);
+}
+
+static const struct file_operations version_fops = {
+ .owner = THIS_MODULE,
+ .read = version_read,
+};
+
+#if defined(DEBUG)
+static int timeline_debugfs_show(struct seq_file *s, void *private_data)
+{
+ struct mali_session_data *session, *tmp;
+ u32 session_seq = 1;
+
+ seq_printf(s, "timeline system info: \n=================\n\n");
+
+ mali_session_lock();
+ MALI_SESSION_FOREACH(session, tmp, link) {
+ seq_printf(s, "session %d <%p> start:\n", session_seq, session);
+ mali_timeline_debug_print_system(session->timeline_system, s);
+ seq_printf(s, "session %d end\n\n\n", session_seq++);
+ }
+ mali_session_unlock();
+
+ return 0;
+}
+
+static int timeline_debugfs_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, timeline_debugfs_show, inode->i_private);
+}
+
+static const struct file_operations timeline_dump_fops = {
+ .owner = THIS_MODULE,
+ .open = timeline_debugfs_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release
+};
+#endif
+
+int mali_sysfs_register(const char *mali_dev_name)
+{
+ mali_debugfs_dir = debugfs_create_dir(mali_dev_name, NULL);
+ if (ERR_PTR(-ENODEV) == mali_debugfs_dir) {
+ /* Debugfs not supported. */
+ mali_debugfs_dir = NULL;
+ } else {
+ if (NULL != mali_debugfs_dir) {
+ /* Debugfs directory created successfully; create files now */
+ struct dentry *mali_power_dir;
+ struct dentry *mali_gp_dir;
+ struct dentry *mali_pp_dir;
+ struct dentry *mali_l2_dir;
+ struct dentry *mali_profiling_dir;
+
+ debugfs_create_file("version", 0400, mali_debugfs_dir, NULL, &version_fops);
+
+ mali_power_dir = debugfs_create_dir("power", mali_debugfs_dir);
+ if (mali_power_dir != NULL) {
+ debugfs_create_file("always_on", 0600, mali_power_dir, NULL, &power_always_on_fops);
+ debugfs_create_file("power_events", 0200, mali_power_dir, NULL, &power_power_events_fops);
+ }
+
+ mali_gp_dir = debugfs_create_dir("gp", mali_debugfs_dir);
+ if (mali_gp_dir != NULL) {
+ u32 num_groups;
+ long i;
+
+ num_groups = mali_group_get_glob_num_groups();
+ for (i = 0; i < num_groups; i++) {
+ struct mali_group *group = mali_group_get_glob_group(i);
+
+ struct mali_gp_core *gp_core = mali_group_get_gp_core(group);
+ if (NULL != gp_core) {
+ struct dentry *mali_gp_gpx_dir;
+ mali_gp_gpx_dir = debugfs_create_dir("gp0", mali_gp_dir);
+ if (NULL != mali_gp_gpx_dir) {
+ debugfs_create_file("base_addr", 0400, mali_gp_gpx_dir, &gp_core->hw_core, &hw_core_base_addr_fops);
+ debugfs_create_file("enabled", 0600, mali_gp_gpx_dir, group, &group_enabled_fops);
+ }
+ break; /* no need to look for any other GP cores */
+ }
+
+ }
+ }
+
+ mali_pp_dir = debugfs_create_dir("pp", mali_debugfs_dir);
+ if (mali_pp_dir != NULL) {
+ u32 num_groups;
+ long i;
+
+ debugfs_create_file("num_cores_total", 0400, mali_pp_dir, NULL, &pp_num_cores_total_fops);
+ debugfs_create_file("num_cores_enabled", 0600, mali_pp_dir, NULL, &pp_num_cores_enabled_fops);
+ debugfs_create_file("core_scaling_enabled", 0600, mali_pp_dir, NULL, &pp_core_scaling_enabled_fops);
+
+ num_groups = mali_group_get_glob_num_groups();
+ for (i = 0; i < num_groups; i++) {
+ struct mali_group *group = mali_group_get_glob_group(i);
+
+ struct mali_pp_core *pp_core = mali_group_get_pp_core(group);
+ if (NULL != pp_core) {
+ char buf[16];
+ struct dentry *mali_pp_ppx_dir;
+ _mali_osk_snprintf(buf, sizeof(buf), "pp%u", mali_pp_core_get_id(pp_core));
+ mali_pp_ppx_dir = debugfs_create_dir(buf, mali_pp_dir);
+ if (NULL != mali_pp_ppx_dir) {
+ debugfs_create_file("base_addr", 0400, mali_pp_ppx_dir, &pp_core->hw_core, &hw_core_base_addr_fops);
+ if (!mali_group_is_virtual(group)) {
+ debugfs_create_file("enabled", 0600, mali_pp_ppx_dir, group, &group_enabled_fops);
+ }
+ }
+ }
+ }
+ }
+
+ mali_l2_dir = debugfs_create_dir("l2", mali_debugfs_dir);
+ if (mali_l2_dir != NULL) {
+ struct dentry *mali_l2_all_dir;
+ u32 l2_id;
+ struct mali_l2_cache_core *l2_cache;
+
+ mali_l2_all_dir = debugfs_create_dir("all", mali_l2_dir);
+ if (mali_l2_all_dir != NULL) {
+ debugfs_create_file("counter_src0", 0200, mali_l2_all_dir, NULL, &l2_all_counter_src0_fops);
+ debugfs_create_file("counter_src1", 0200, mali_l2_all_dir, NULL, &l2_all_counter_src1_fops);
+ }
+
+ l2_id = 0;
+ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id);
+ while (NULL != l2_cache) {
+ char buf[16];
+ struct dentry *mali_l2_l2x_dir;
+ _mali_osk_snprintf(buf, sizeof(buf), "l2%u", l2_id);
+ mali_l2_l2x_dir = debugfs_create_dir(buf, mali_l2_dir);
+ if (NULL != mali_l2_l2x_dir) {
+ debugfs_create_file("counter_src0", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_src0_fops);
+ debugfs_create_file("counter_src1", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_src1_fops);
+ debugfs_create_file("counter_val0", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_val0_fops);
+ debugfs_create_file("counter_val1", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_val1_fops);
+ debugfs_create_file("base_addr", 0400, mali_l2_l2x_dir, &l2_cache->hw_core, &hw_core_base_addr_fops);
+ }
+
+ /* try next L2 */
+ l2_id++;
+ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id);
+ }
+ }
+
+ debugfs_create_file("gpu_memory", 0444, mali_debugfs_dir, NULL, &memory_usage_fops);
+
+ debugfs_create_file("utilization_gp_pp", 0400, mali_debugfs_dir, NULL, &utilization_gp_pp_fops);
+ debugfs_create_file("utilization_gp", 0400, mali_debugfs_dir, NULL, &utilization_gp_fops);
+ debugfs_create_file("utilization_pp", 0400, mali_debugfs_dir, NULL, &utilization_pp_fops);
+
+ mali_profiling_dir = debugfs_create_dir("profiling", mali_debugfs_dir);
+ if (mali_profiling_dir != NULL) {
+ u32 max_sub_jobs;
+ long i;
+ struct dentry *mali_profiling_gp_dir;
+ struct dentry *mali_profiling_pp_dir;
+#if defined(CONFIG_MALI400_INTERNAL_PROFILING)
+ struct dentry *mali_profiling_proc_dir;
+#endif
+ /*
+ * Create directory where we can set GP HW counters.
+ */
+ mali_profiling_gp_dir = debugfs_create_dir("gp", mali_profiling_dir);
+ if (mali_profiling_gp_dir != NULL) {
+ debugfs_create_file("counter_src0", 0600, mali_profiling_gp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_GP(0), &profiling_counter_src_fops);
+ debugfs_create_file("counter_src1", 0600, mali_profiling_gp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_GP(1), &profiling_counter_src_fops);
+ }
+
+ /*
+ * Create directory where we can set PP HW counters.
+ * Possible override with specific HW counters for a particular sub job
+ * (Disable core scaling before using the override!)
+ */
+ mali_profiling_pp_dir = debugfs_create_dir("pp", mali_profiling_dir);
+ if (mali_profiling_pp_dir != NULL) {
+ debugfs_create_file("counter_src0", 0600, mali_profiling_pp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_PP(0), &profiling_counter_src_fops);
+ debugfs_create_file("counter_src1", 0600, mali_profiling_pp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_PP(1), &profiling_counter_src_fops);
+ }
+
+ max_sub_jobs = mali_executor_get_num_cores_total();
+ for (i = 0; i < max_sub_jobs; i++) {
+ char buf[16];
+ struct dentry *mali_profiling_pp_x_dir;
+ _mali_osk_snprintf(buf, sizeof(buf), "%u", i);
+ mali_profiling_pp_x_dir = debugfs_create_dir(buf, mali_profiling_pp_dir);
+ if (NULL != mali_profiling_pp_x_dir) {
+ debugfs_create_file("counter_src0",
+ 0600, mali_profiling_pp_x_dir,
+ (void *)PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(0, i),
+ &profiling_counter_src_fops);
+ debugfs_create_file("counter_src1",
+ 0600, mali_profiling_pp_x_dir,
+ (void *)PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(1, i),
+ &profiling_counter_src_fops);
+ }
+ }
+
+#if defined(CONFIG_MALI400_INTERNAL_PROFILING)
+ mali_profiling_proc_dir = debugfs_create_dir("proc", mali_profiling_dir);
+ if (mali_profiling_proc_dir != NULL) {
+ struct dentry *mali_profiling_proc_default_dir = debugfs_create_dir("default", mali_profiling_proc_dir);
+ if (mali_profiling_proc_default_dir != NULL) {
+ debugfs_create_file("enable", 0600, mali_profiling_proc_default_dir, (void *)_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, &user_settings_fops);
+ }
+ }
+ debugfs_create_file("record", 0600, mali_profiling_dir, NULL, &profiling_record_fops);
+ debugfs_create_file("events", 0400, mali_profiling_dir, NULL, &profiling_events_fops);
+ debugfs_create_file("events_human_readable", 0400, mali_profiling_dir, NULL, &profiling_events_human_readable_fops);
+#endif
+ }
+
+#if MALI_STATE_TRACKING
+ debugfs_create_file("state_dump", 0400, mali_debugfs_dir, NULL, &mali_seq_internal_state_fops);
+#endif
+
+#if defined(DEBUG)
+ debugfs_create_file("timeline_dump", 0400, mali_debugfs_dir, NULL, &timeline_dump_fops);
+#endif
+ if (mali_sysfs_user_settings_register()) {
+ /* Failed to create the debugfs entries for the user settings DB. */
+ MALI_DEBUG_PRINT(2, ("Failed to create user setting debugfs files. Ignoring...\n"));
+ }
+ }
+ }
+
+ /* Success! */
+ return 0;
+}
+
+int mali_sysfs_unregister(void)
+{
+ if (NULL != mali_debugfs_dir) {
+ debugfs_remove_recursive(mali_debugfs_dir);
+ }
+ return 0;
+}
+
+#else /* MALI_LICENSE_IS_GPL */
+
+/* Dummy implementations for non-GPL */
+
+int mali_sysfs_register(struct mali_dev *device, dev_t dev, const char *mali_dev_name)
+{
+ return 0;
+}
+
+int mali_sysfs_unregister(void)
+{
+ return 0;
+}
+
+#endif /* MALI_LICENSE_IS_GPL */
diff --git a/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.h b/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.h
new file mode 100644
index 000000000000..a36a0cea9972
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2011-2013, 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_KERNEL_SYSFS_H__
+#define __MALI_KERNEL_SYSFS_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <linux/device.h>
+
+#define MALI_PROC_DIR "driver/mali"
+
+int mali_sysfs_register(const char *mali_dev_name);
+int mali_sysfs_unregister(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __MALI_KERNEL_LINUX_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_linux_trace.h b/drivers/gpu/arm/utgard/linux/mali_linux_trace.h
new file mode 100644
index 000000000000..c6cd2bfb7217
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_linux_trace.h
@@ -0,0 +1,162 @@
+/*
+ * Copyright (C) 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#if !defined (MALI_LINUX_TRACE_H) || defined (TRACE_HEADER_MULTI_READ)
+#define MALI_LINUX_TRACE_H
+
+#include <linux/types.h>
+
+#include <linux/stringify.h>
+#include <linux/tracepoint.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM mali
+#define TRACE_SYSTEM_STRING __stringfy(TRACE_SYSTEM)
+
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE mali_linux_trace
+
+/**
+ * Define the tracepoint used to communicate the status of a GPU. Called
+ * when a GPU turns on or turns off.
+ *
+ * @param event_id The type of the event. This parameter is a bitfield
+ * encoding the type of the event.
+ *
+ * @param d0 First data parameter.
+ * @param d1 Second data parameter.
+ * @param d2 Third data parameter.
+ * @param d3 Fourth data parameter.
+ * @param d4 Fifth data parameter.
+ */
+TRACE_EVENT(mali_timeline_event,
+
+ TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1,
+ unsigned int d2, unsigned int d3, unsigned int d4),
+
+ TP_ARGS(event_id, d0, d1, d2, d3, d4),
+
+ TP_STRUCT__entry(
+ __field(unsigned int, event_id)
+ __field(unsigned int, d0)
+ __field(unsigned int, d1)
+ __field(unsigned int, d2)
+ __field(unsigned int, d3)
+ __field(unsigned int, d4)
+ ),
+
+ TP_fast_assign(
+ __entry->event_id = event_id;
+ __entry->d0 = d0;
+ __entry->d1 = d1;
+ __entry->d2 = d2;
+ __entry->d3 = d3;
+ __entry->d4 = d4;
+ ),
+
+ TP_printk("event=%d", __entry->event_id)
+ );
+
+/**
+ * Define a tracepoint used to regsiter the value of a hardware counter.
+ * Hardware counters belonging to the vertex or fragment processor are
+ * reported via this tracepoint each frame, whilst L2 cache hardware
+ * counters are reported continuously.
+ *
+ * @param counter_id The counter ID.
+ * @param value The value of the counter.
+ */
+TRACE_EVENT(mali_hw_counter,
+
+ TP_PROTO(unsigned int counter_id, unsigned int value),
+
+ TP_ARGS(counter_id, value),
+
+ TP_STRUCT__entry(
+ __field(unsigned int, counter_id)
+ __field(unsigned int, value)
+ ),
+
+ TP_fast_assign(
+ __entry->counter_id = counter_id;
+ ),
+
+ TP_printk("event %d = %d", __entry->counter_id, __entry->value)
+ );
+
+/**
+ * Define a tracepoint used to send a bundle of software counters.
+ *
+ * @param counters The bundle of counters.
+ */
+TRACE_EVENT(mali_sw_counters,
+
+ TP_PROTO(pid_t pid, pid_t tid, void *surface_id, unsigned int *counters),
+
+ TP_ARGS(pid, tid, surface_id, counters),
+
+ TP_STRUCT__entry(
+ __field(pid_t, pid)
+ __field(pid_t, tid)
+ __field(void *, surface_id)
+ __field(unsigned int *, counters)
+ ),
+
+ TP_fast_assign(
+ __entry->pid = pid;
+ __entry->tid = tid;
+ __entry->surface_id = surface_id;
+ __entry->counters = counters;
+ ),
+
+ TP_printk("counters were %s", __entry->counters == NULL ? "NULL" : "not NULL")
+ );
+
+/**
+ * Define a tracepoint used to gather core activity for systrace
+ * @param pid The process id for which the core activity originates from
+ * @param active If the core is active (1) or not (0)
+ * @param core_type The type of core active, either GP (1) or PP (0)
+ * @param core_id The core id that is active for the core_type
+ * @param frame_builder_id The frame builder id associated with this core activity
+ * @param flush_id The flush id associated with this core activity
+ */
+TRACE_EVENT(mali_core_active,
+
+ TP_PROTO(pid_t pid, unsigned int active, unsigned int core_type, unsigned int core_id, unsigned int frame_builder_id, unsigned int flush_id),
+
+ TP_ARGS(pid, active, core_type, core_id, frame_builder_id, flush_id),
+
+ TP_STRUCT__entry(
+ __field(pid_t, pid)
+ __field(unsigned int, active)
+ __field(unsigned int, core_type)
+ __field(unsigned int, core_id)
+ __field(unsigned int, frame_builder_id)
+ __field(unsigned int, flush_id)
+ ),
+
+ TP_fast_assign(
+ __entry->pid = pid;
+ __entry->active = active;
+ __entry->core_type = core_type;
+ __entry->core_id = core_id;
+ __entry->frame_builder_id = frame_builder_id;
+ __entry->flush_id = flush_id;
+ ),
+
+ TP_printk("%s|%d|%s%i:%x|%d", __entry->active ? "S" : "F", __entry->pid, __entry->core_type ? "GP" : "PP", __entry->core_id, __entry->flush_id, __entry->frame_builder_id)
+ );
+
+#endif /* MALI_LINUX_TRACE_H */
+
+/* This part must exist outside the header guard. */
+#include <trace/define_trace.h>
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory.c b/drivers/gpu/arm/utgard/linux/mali_memory.c
new file mode 100644
index 000000000000..c45f6ee25887
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory.c
@@ -0,0 +1,510 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/fs.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+#include <linux/idr.h>
+
+#include "mali_osk.h"
+#include "mali_executor.h"
+
+#include "mali_memory.h"
+#include "mali_memory_os_alloc.h"
+#include "mali_memory_block_alloc.h"
+#include "mali_memory_util.h"
+#include "mali_memory_virtual.h"
+#include "mali_memory_manager.h"
+#include "mali_memory_cow.h"
+#include "mali_memory_swap_alloc.h"
+#include "mali_memory_defer_bind.h"
+
+extern unsigned int mali_dedicated_mem_size;
+extern unsigned int mali_shared_mem_size;
+
+#define MALI_VM_NUM_FAULT_PREFETCH (0x8)
+
+static void mali_mem_vma_open(struct vm_area_struct *vma)
+{
+ mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data;
+ MALI_DEBUG_PRINT(4, ("Open called on vma %p\n", vma));
+
+ /* If need to share the allocation, add ref_count here */
+ mali_allocation_ref(alloc);
+ return;
+}
+static void mali_mem_vma_close(struct vm_area_struct *vma)
+{
+ /* If need to share the allocation, unref ref_count here */
+ mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data;
+
+ mali_allocation_unref(&alloc);
+ vma->vm_private_data = NULL;
+}
+
+static int mali_mem_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+ mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data;
+ mali_mem_backend *mem_bkend = NULL;
+ int ret;
+ int prefetch_num = MALI_VM_NUM_FAULT_PREFETCH;
+
+ unsigned long address = (unsigned long)vmf->virtual_address;
+ MALI_DEBUG_ASSERT(alloc->backend_handle);
+ MALI_DEBUG_ASSERT((unsigned long)alloc->cpu_mapping.addr <= address);
+
+ /* Get backend memory & Map on CPU */
+ mutex_lock(&mali_idr_mutex);
+ if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) {
+ MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n"));
+ mutex_unlock(&mali_idr_mutex);
+ return VM_FAULT_SIGBUS;
+ }
+ mutex_unlock(&mali_idr_mutex);
+ MALI_DEBUG_ASSERT(mem_bkend->type == alloc->type);
+
+ if ((mem_bkend->type == MALI_MEM_COW && (MALI_MEM_BACKEND_FLAG_SWAP_COWED !=
+ (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) &&
+ (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE)) {
+ /*check if use page fault to do COW*/
+ MALI_DEBUG_PRINT(4, ("mali_vma_fault: do cow allocate on demand!, address=0x%x\n", address));
+ mutex_lock(&mem_bkend->mutex);
+ ret = mali_mem_cow_allocate_on_demand(mem_bkend,
+ (address - vma->vm_start) / PAGE_SIZE);
+ mutex_unlock(&mem_bkend->mutex);
+
+ if (ret != _MALI_OSK_ERR_OK) {
+ return VM_FAULT_OOM;
+ }
+ prefetch_num = 1;
+
+ /* handle COW modified range cpu mapping
+ we zap the mapping in cow_modify_range, it will trigger page fault
+ when CPU access it, so here we map it to CPU*/
+ mutex_lock(&mem_bkend->mutex);
+ ret = mali_mem_cow_cpu_map_pages_locked(mem_bkend, vma, address, prefetch_num);
+ mutex_unlock(&mem_bkend->mutex);
+
+ if (unlikely(ret != _MALI_OSK_ERR_OK)) {
+ return VM_FAULT_SIGBUS;
+ }
+ } else if ((mem_bkend->type == MALI_MEM_SWAP) ||
+ (mem_bkend->type == MALI_MEM_COW && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) {
+ u32 offset_in_bkend = (address - vma->vm_start) / PAGE_SIZE;
+ int ret = _MALI_OSK_ERR_OK;
+
+ mutex_lock(&mem_bkend->mutex);
+ if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE) {
+ ret = mali_mem_swap_cow_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page);
+ } else {
+ ret = mali_mem_swap_allocate_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page);
+ }
+ mutex_unlock(&mem_bkend->mutex);
+
+ if (ret != _MALI_OSK_ERR_OK) {
+ MALI_DEBUG_PRINT(2, ("Mali swap memory page fault process failed, address=0x%x\n", address));
+ return VM_FAULT_OOM;
+ } else {
+ return VM_FAULT_LOCKED;
+ }
+ } else {
+ MALI_DEBUG_ASSERT(0);
+ /*NOT support yet*/
+ }
+ return VM_FAULT_NOPAGE;
+}
+
+static struct vm_operations_struct mali_kernel_vm_ops = {
+ .open = mali_mem_vma_open,
+ .close = mali_mem_vma_close,
+ .fault = mali_mem_vma_fault,
+};
+
+
+/** @ map mali allocation to CPU address
+*
+* Supported backend types:
+* --MALI_MEM_OS
+* -- need to add COW?
+ *Not supported backend types:
+* -_MALI_MEMORY_BIND_BACKEND_UMP
+* -_MALI_MEMORY_BIND_BACKEND_DMA_BUF
+* -_MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY
+*
+*/
+int mali_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+ struct mali_session_data *session;
+ mali_mem_allocation *mali_alloc = NULL;
+ u32 mali_addr = vma->vm_pgoff << PAGE_SHIFT;
+ struct mali_vma_node *mali_vma_node = NULL;
+ mali_mem_backend *mem_bkend = NULL;
+ int ret = -EFAULT;
+
+ session = (struct mali_session_data *)filp->private_data;
+ if (NULL == session) {
+ MALI_PRINT_ERROR(("mmap called without any session data available\n"));
+ return -EFAULT;
+ }
+
+ MALI_DEBUG_PRINT(4, ("MMap() handler: start=0x%08X, phys=0x%08X, size=0x%08X vma->flags 0x%08x\n",
+ (unsigned int)vma->vm_start, (unsigned int)(vma->vm_pgoff << PAGE_SHIFT),
+ (unsigned int)(vma->vm_end - vma->vm_start), vma->vm_flags));
+
+ /* Operations used on any memory system */
+ /* do not need to anything in vm open/close now */
+
+ /* find mali allocation structure by vaddress*/
+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
+ if (likely(mali_vma_node)) {
+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
+ MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start);
+ if (unlikely(mali_addr != mali_vma_node->vm_node.start)) {
+ /* only allow to use start address for mmap */
+ MALI_DEBUG_PRINT(1, ("mali_addr != mali_vma_node->vm_node.start\n"));
+ return -EFAULT;
+ }
+ } else {
+ MALI_DEBUG_ASSERT(NULL == mali_vma_node);
+ return -EFAULT;
+ }
+
+ mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start;
+
+ if (mali_alloc->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) {
+ MALI_DEBUG_PRINT(1, ("ERROR : trying to access varying memory by CPU!\n"));
+ return -EFAULT;
+ }
+
+ /* Get backend memory & Map on CPU */
+ mutex_lock(&mali_idr_mutex);
+ if (!(mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle))) {
+ MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n"));
+ mutex_unlock(&mali_idr_mutex);
+ return -EFAULT;
+ }
+ mutex_unlock(&mali_idr_mutex);
+
+ if (!(MALI_MEM_SWAP == mali_alloc->type ||
+ (MALI_MEM_COW == mali_alloc->type && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) {
+ /* Set some bits which indicate that, the memory is IO memory, meaning
+ * that no paging is to be performed and the memory should not be
+ * included in crash dumps. And that the memory is reserved, meaning
+ * that it's present and can never be paged out (see also previous
+ * entry)
+ */
+ vma->vm_flags |= VM_IO;
+ vma->vm_flags |= VM_DONTCOPY;
+ vma->vm_flags |= VM_PFNMAP;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0)
+ vma->vm_flags |= VM_RESERVED;
+#else
+ vma->vm_flags |= VM_DONTDUMP;
+ vma->vm_flags |= VM_DONTEXPAND;
+#endif
+ } else if (MALI_MEM_SWAP == mali_alloc->type) {
+ vma->vm_pgoff = mem_bkend->start_idx;
+ }
+
+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+ vma->vm_ops = &mali_kernel_vm_ops;
+
+ mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start;
+
+ /* If it's a copy-on-write mapping, map to read only */
+ if (!(vma->vm_flags & VM_WRITE)) {
+ MALI_DEBUG_PRINT(4, ("mmap allocation with read only !\n"));
+ /* add VM_WRITE for do_page_fault will check this when a write fault */
+ vma->vm_flags |= VM_WRITE | VM_READ;
+ vma->vm_page_prot = PAGE_READONLY;
+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+ mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE;
+ goto out;
+ }
+
+ if (mem_bkend->type == MALI_MEM_OS) {
+ ret = mali_mem_os_cpu_map(mem_bkend, vma);
+ } else if (mem_bkend->type == MALI_MEM_COW &&
+ (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) {
+ ret = mali_mem_cow_cpu_map(mem_bkend, vma);
+ } else if (mem_bkend->type == MALI_MEM_BLOCK) {
+ ret = mali_mem_block_cpu_map(mem_bkend, vma);
+ } else if ((mem_bkend->type == MALI_MEM_SWAP) || (mem_bkend->type == MALI_MEM_COW &&
+ (MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) {
+ /*For swappable memory, CPU page table will be created by page fault handler. */
+ ret = 0;
+ } else {
+ /* Not support yet*/
+ MALI_DEBUG_ASSERT(0);
+ }
+
+ if (ret != 0) {
+ MALI_DEBUG_PRINT(1, ("ret != 0\n"));
+ return -EFAULT;
+ }
+out:
+ MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == mali_alloc->magic);
+
+ vma->vm_private_data = (void *)mali_alloc;
+ mali_alloc->cpu_mapping.vma = vma;
+
+ mali_allocation_ref(mali_alloc);
+
+ return 0;
+}
+
+_mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor)
+{
+ u32 size = descriptor->psize;
+ struct mali_session_data *session = descriptor->session;
+
+ MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic);
+
+ /* Map dma-buf into this session's page tables */
+
+ if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) {
+ size += MALI_MMU_PAGE_SIZE;
+ }
+
+ return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start, size);
+}
+
+_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size)
+{
+ u32 old_size = descriptor->psize;
+ struct mali_session_data *session = descriptor->session;
+
+ MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic);
+
+ if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) {
+ new_size += MALI_MMU_PAGE_SIZE;
+ }
+
+ if (new_size > old_size) {
+ MALI_DEBUG_ASSERT(new_size <= descriptor->mali_vma_node.vm_node.size);
+ return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start + old_size, new_size - old_size);
+ }
+ return _MALI_OSK_ERR_OK;
+}
+
+void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags)
+{
+ if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) {
+ size += MALI_MMU_PAGE_SIZE;
+ }
+
+ /* Umap and flush L2 */
+ mali_mmu_pagedir_unmap(session->page_directory, vaddr, size);
+ mali_executor_zap_all_active(session);
+}
+
+u32 _mali_ukk_report_memory_usage(void)
+{
+ u32 sum = 0;
+
+ if (MALI_TRUE == mali_memory_have_dedicated_memory()) {
+ sum += mali_mem_block_allocator_stat();
+ }
+
+ sum += mali_mem_os_stat();
+
+ return sum;
+}
+
+u32 _mali_ukk_report_total_memory_size(void)
+{
+ return mali_dedicated_mem_size + mali_shared_mem_size;
+}
+
+
+/**
+ * Per-session memory descriptor mapping table sizes
+ */
+#define MALI_MEM_DESCRIPTORS_INIT 64
+#define MALI_MEM_DESCRIPTORS_MAX 65536
+
+_mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session_data)
+{
+ MALI_DEBUG_PRINT(5, ("Memory session begin\n"));
+
+ session_data->memory_lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED,
+ _MALI_OSK_LOCK_ORDER_MEM_SESSION);
+
+ if (NULL == session_data->memory_lock) {
+ MALI_ERROR(_MALI_OSK_ERR_FAULT);
+ }
+
+ mali_memory_manager_init(&session_data->allocation_mgr);
+
+ MALI_DEBUG_PRINT(5, ("MMU session begin: success\n"));
+ MALI_SUCCESS;
+}
+
+void mali_memory_session_end(struct mali_session_data *session)
+{
+ MALI_DEBUG_PRINT(3, ("MMU session end\n"));
+
+ if (NULL == session) {
+ MALI_DEBUG_PRINT(1, ("No session data found during session end\n"));
+ return;
+ }
+ /* free allocation */
+ mali_free_session_allocations(session);
+ /* do some check in unint*/
+ mali_memory_manager_uninit(&session->allocation_mgr);
+
+ /* Free the lock */
+ _mali_osk_mutex_term(session->memory_lock);
+
+ return;
+}
+
+_mali_osk_errcode_t mali_memory_initialize(void)
+{
+ _mali_osk_errcode_t err;
+
+ idr_init(&mali_backend_idr);
+ mutex_init(&mali_idr_mutex);
+
+ err = mali_mem_swap_init();
+ if (err != _MALI_OSK_ERR_OK) {
+ return err;
+ }
+ err = mali_mem_os_init();
+ if (_MALI_OSK_ERR_OK == err) {
+ err = mali_mem_defer_bind_manager_init();
+ }
+
+ return err;
+}
+
+void mali_memory_terminate(void)
+{
+ mali_mem_swap_term();
+ mali_mem_defer_bind_manager_destory();
+ mali_mem_os_term();
+ if (mali_memory_have_dedicated_memory()) {
+ mali_mem_block_allocator_destroy();
+ }
+}
+
+
+struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type)
+{
+ mali_page_node *page_node = NULL;
+
+ page_node = kzalloc(sizeof(mali_page_node), GFP_KERNEL);
+ MALI_DEBUG_ASSERT(NULL != page_node);
+
+ if (page_node) {
+ page_node->type = type;
+ INIT_LIST_HEAD(&page_node->list);
+ }
+
+ return page_node;
+}
+
+void _mali_page_node_ref(struct mali_page_node *node)
+{
+ if (node->type == MALI_PAGE_NODE_OS) {
+ /* add ref to this page */
+ get_page(node->page);
+ } else if (node->type == MALI_PAGE_NODE_BLOCK) {
+ mali_mem_block_add_ref(node);
+ } else if (node->type == MALI_PAGE_NODE_SWAP) {
+ atomic_inc(&node->swap_it->ref_count);
+ } else
+ MALI_DEBUG_ASSERT(0);
+}
+
+void _mali_page_node_unref(struct mali_page_node *node)
+{
+ if (node->type == MALI_PAGE_NODE_OS) {
+ /* unref to this page */
+ put_page(node->page);
+ } else if (node->type == MALI_PAGE_NODE_BLOCK) {
+ mali_mem_block_dec_ref(node);
+ } else
+ MALI_DEBUG_ASSERT(0);
+}
+
+
+void _mali_page_node_add_page(struct mali_page_node *node, struct page *page)
+{
+ MALI_DEBUG_ASSERT(MALI_PAGE_NODE_OS == node->type);
+ node->page = page;
+}
+
+
+void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item)
+{
+ MALI_DEBUG_ASSERT(MALI_PAGE_NODE_SWAP == node->type);
+ node->swap_it = item;
+}
+
+void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item)
+{
+ MALI_DEBUG_ASSERT(MALI_PAGE_NODE_BLOCK == node->type);
+ node->blk_it = item;
+}
+
+
+int _mali_page_node_get_ref_count(struct mali_page_node *node)
+{
+ if (node->type == MALI_PAGE_NODE_OS) {
+ /* get ref count of this page */
+ return page_count(node->page);
+ } else if (node->type == MALI_PAGE_NODE_BLOCK) {
+ return mali_mem_block_get_ref_count(node);
+ } else if (node->type == MALI_PAGE_NODE_SWAP) {
+ return atomic_read(&node->swap_it->ref_count);
+ } else {
+ MALI_DEBUG_ASSERT(0);
+ }
+ return -1;
+}
+
+
+dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node)
+{
+ if (node->type == MALI_PAGE_NODE_OS) {
+ return page_private(node->page);
+ } else if (node->type == MALI_PAGE_NODE_BLOCK) {
+ return _mali_blk_item_get_phy_addr(node->blk_it);
+ } else if (node->type == MALI_PAGE_NODE_SWAP) {
+ return node->swap_it->dma_addr;
+ } else {
+ MALI_DEBUG_ASSERT(0);
+ }
+ return 0;
+}
+
+
+unsigned long _mali_page_node_get_pfn(struct mali_page_node *node)
+{
+ if (node->type == MALI_PAGE_NODE_OS) {
+ return page_to_pfn(node->page);
+ } else if (node->type == MALI_PAGE_NODE_BLOCK) {
+ /* get phy addr for BLOCK page*/
+ return _mali_blk_item_get_pfn(node->blk_it);
+ } else if (node->type == MALI_PAGE_NODE_SWAP) {
+ return page_to_pfn(node->swap_it->page);
+ } else {
+ MALI_DEBUG_ASSERT(0);
+ }
+ return 0;
+}
+
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory.h b/drivers/gpu/arm/utgard/linux/mali_memory.h
new file mode 100644
index 000000000000..3140c6c98d2c
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory.h
@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_MEMORY_H__
+#define __MALI_MEMORY_H__
+
+#include "mali_osk.h"
+#include "mali_session.h"
+
+#include <linux/list.h>
+#include <linux/mm.h>
+
+#include "mali_memory_types.h"
+#include "mali_memory_os_alloc.h"
+
+_mali_osk_errcode_t mali_memory_initialize(void);
+void mali_memory_terminate(void);
+
+/** @brief Allocate a page table page
+ *
+ * Allocate a page for use as a page directory or page table. The page is
+ * mapped into kernel space.
+ *
+ * @return _MALI_OSK_ERR_OK on success, otherwise an error code
+ * @param table_page GPU pointer to the allocated page
+ * @param mapping CPU pointer to the mapping of the allocated page
+ */
+MALI_STATIC_INLINE _mali_osk_errcode_t
+mali_mmu_get_table_page(mali_dma_addr *table_page, mali_io_address *mapping)
+{
+ return mali_mem_os_get_table_page(table_page, mapping);
+}
+
+/** @brief Release a page table page
+ *
+ * Release a page table page allocated through \a mali_mmu_get_table_page
+ *
+ * @param pa the GPU address of the page to release
+ */
+MALI_STATIC_INLINE void
+mali_mmu_release_table_page(mali_dma_addr phys, void *virt)
+{
+ mali_mem_os_release_table_page(phys, virt);
+}
+
+/** @brief mmap function
+ *
+ * mmap syscalls on the Mali device node will end up here.
+ *
+ * This function allocates Mali memory and maps it on CPU and Mali.
+ */
+int mali_mmap(struct file *filp, struct vm_area_struct *vma);
+
+/** @brief Start a new memory session
+ *
+ * Called when a process opens the Mali device node.
+ *
+ * @param session Pointer to session to initialize
+ */
+_mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session);
+
+/** @brief Close a memory session
+ *
+ * Called when a process closes the Mali device node.
+ *
+ * Memory allocated by the session will be freed
+ *
+ * @param session Pointer to the session to terminate
+ */
+void mali_memory_session_end(struct mali_session_data *session);
+
+/** @brief Prepare Mali page tables for mapping
+ *
+ * This function will prepare the Mali page tables for mapping the memory
+ * described by \a descriptor.
+ *
+ * Page tables will be reference counted and allocated, if not yet present.
+ *
+ * @param descriptor Pointer to the memory descriptor to the mapping
+ */
+_mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor);
+
+/** @brief Resize Mali page tables for mapping
+ *
+ * This function will Resize the Mali page tables for mapping the memory
+ * described by \a descriptor.
+ *
+ * Page tables will be reference counted and allocated, if not yet present.
+ *
+ * @param descriptor Pointer to the memory descriptor to the mapping
+ * @param new_size The new size of descriptor
+ */
+_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size);
+
+/** @brief Free Mali page tables for mapping
+ *
+ * This function will unmap pages from Mali memory and free the page tables
+ * that are now unused.
+ *
+ * The updated pages in the Mali L2 cache will be invalidated, and the MMU TLBs will be zapped if necessary.
+ *
+ * @param descriptor Pointer to the memory descriptor to unmap
+ */
+void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags);
+
+/** @brief Parse resource and prepare the OS memory allocator
+ *
+ * @param size Maximum size to allocate for Mali GPU.
+ * @return _MALI_OSK_ERR_OK on success, otherwise failure.
+ */
+_mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size);
+
+/** @brief Parse resource and prepare the dedicated memory allocator
+ *
+ * @param start Physical start address of dedicated Mali GPU memory.
+ * @param size Size of dedicated Mali GPU memory.
+ * @return _MALI_OSK_ERR_OK on success, otherwise failure.
+ */
+_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size);
+
+
+struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type);
+
+void _mali_page_node_ref(struct mali_page_node *node);
+void _mali_page_node_unref(struct mali_page_node *node);
+void _mali_page_node_add_page(struct mali_page_node *node, struct page *page);
+
+void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item);
+
+void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item);
+
+int _mali_page_node_get_ref_count(struct mali_page_node *node);
+dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node);
+unsigned long _mali_page_node_get_pfn(struct mali_page_node *node);
+
+#endif /* __MALI_MEMORY_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.c b/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.c
new file mode 100644
index 000000000000..453ddda3080f
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.c
@@ -0,0 +1,362 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include "mali_kernel_common.h"
+#include "mali_memory.h"
+#include "mali_memory_block_alloc.h"
+#include "mali_osk.h"
+#include <linux/mutex.h>
+
+
+static mali_block_allocator *mali_mem_block_gobal_allocator = NULL;
+
+unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item)
+{
+ return (item->phy_addr & ~(MALI_BLOCK_REF_MASK));
+}
+
+
+unsigned long _mali_blk_item_get_pfn(mali_block_item *item)
+{
+ return (item->phy_addr / MALI_BLOCK_SIZE);
+}
+
+
+u32 mali_mem_block_get_ref_count(mali_page_node *node)
+{
+ MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK);
+ return (node->blk_it->phy_addr & MALI_BLOCK_REF_MASK);
+}
+
+
+/* Increase the refence count
+* It not atomic, so it need to get sp_lock before call this function
+*/
+
+u32 mali_mem_block_add_ref(mali_page_node *node)
+{
+ MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK);
+ MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) < MALI_BLOCK_MAX_REF_COUNT);
+ return (node->blk_it->phy_addr++ & MALI_BLOCK_REF_MASK);
+}
+
+/* Decase the refence count
+* It not atomic, so it need to get sp_lock before call this function
+*/
+u32 mali_mem_block_dec_ref(mali_page_node *node)
+{
+ MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK);
+ MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) > 0);
+ return (node->blk_it->phy_addr-- & MALI_BLOCK_REF_MASK);
+}
+
+
+static mali_block_allocator *mali_mem_block_allocator_create(u32 base_address, u32 size)
+{
+ mali_block_allocator *info;
+ u32 usable_size;
+ u32 num_blocks;
+ mali_page_node *m_node;
+ mali_block_item *mali_blk_items = NULL;
+ int i = 0;
+
+ usable_size = size & ~(MALI_BLOCK_SIZE - 1);
+ MALI_DEBUG_PRINT(3, ("Mali block allocator create for region starting at 0x%08X length 0x%08X\n", base_address, size));
+ MALI_DEBUG_PRINT(4, ("%d usable bytes\n", usable_size));
+ num_blocks = usable_size / MALI_BLOCK_SIZE;
+ MALI_DEBUG_PRINT(4, ("which becomes %d blocks\n", num_blocks));
+
+ if (usable_size == 0) {
+ MALI_DEBUG_PRINT(1, ("Memory block of size %d is unusable\n", size));
+ return NULL;
+ }
+
+ info = _mali_osk_calloc(1, sizeof(mali_block_allocator));
+ if (NULL != info) {
+ INIT_LIST_HEAD(&info->free);
+ spin_lock_init(&info->sp_lock);
+ info->total_num = num_blocks;
+ mali_blk_items = _mali_osk_calloc(1, sizeof(mali_block_item) * num_blocks);
+
+ if (mali_blk_items) {
+ info->items = mali_blk_items;
+ /* add blocks(4k size) to free list*/
+ for (i = 0 ; i < num_blocks ; i++) {
+ /* add block information*/
+ mali_blk_items[i].phy_addr = base_address + (i * MALI_BLOCK_SIZE);
+ /* add to free list */
+ m_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK);
+ if (m_node == NULL)
+ goto fail;
+ _mali_page_node_add_block_item(m_node, &(mali_blk_items[i]));
+ list_add_tail(&m_node->list, &info->free);
+ atomic_add(1, &info->free_num);
+ }
+ return info;
+ }
+ }
+fail:
+ mali_mem_block_allocator_destroy();
+ return NULL;
+}
+
+void mali_mem_block_allocator_destroy(void)
+{
+ struct mali_page_node *m_page, *m_tmp;
+ mali_block_allocator *info = mali_mem_block_gobal_allocator;
+ MALI_DEBUG_ASSERT_POINTER(info);
+ MALI_DEBUG_PRINT(4, ("Memory block destroy !\n"));
+
+ if (NULL == info)
+ return;
+
+ list_for_each_entry_safe(m_page, m_tmp , &info->free, list) {
+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK);
+ list_del(&m_page->list);
+ kfree(m_page);
+ }
+
+ _mali_osk_free(info->items);
+ _mali_osk_free(info);
+}
+
+u32 mali_mem_block_release(mali_mem_backend *mem_bkend)
+{
+ mali_mem_allocation *alloc = mem_bkend->mali_allocation;
+ u32 free_pages_nr = 0;
+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK);
+
+ /* Unmap the memory from the mali virtual address space. */
+ mali_mem_block_mali_unmap(alloc);
+ mutex_lock(&mem_bkend->mutex);
+ free_pages_nr = mali_mem_block_free(&mem_bkend->block_mem);
+ mutex_unlock(&mem_bkend->mutex);
+ return free_pages_nr;
+}
+
+
+int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size)
+{
+ struct mali_page_node *m_page, *m_tmp;
+ size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE;
+ mali_block_allocator *info = mali_mem_block_gobal_allocator;
+ MALI_DEBUG_ASSERT_POINTER(info);
+
+ MALI_DEBUG_PRINT(4, ("BLOCK Mem: Allocate size = 0x%x\n", size));
+ /*do some init */
+ INIT_LIST_HEAD(&block_mem->pfns);
+
+ spin_lock(&info->sp_lock);
+ /*check if have enough space*/
+ if (atomic_read(&info->free_num) > page_count) {
+ list_for_each_entry_safe(m_page, m_tmp , &info->free, list) {
+ if (page_count > 0) {
+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK);
+ MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(m_page) == 0);
+ list_move(&m_page->list, &block_mem->pfns);
+ block_mem->count++;
+ atomic_dec(&info->free_num);
+ _mali_page_node_ref(m_page);
+ } else {
+ break;
+ }
+ page_count--;
+ }
+ } else {
+ /* can't allocate from BLOCK memory*/
+ spin_unlock(&info->sp_lock);
+ return -1;
+ }
+
+ spin_unlock(&info->sp_lock);
+ return 0;
+}
+
+u32 mali_mem_block_free(mali_mem_block_mem *block_mem)
+{
+ u32 free_pages_nr = 0;
+
+ free_pages_nr = mali_mem_block_free_list(&block_mem->pfns);
+ MALI_DEBUG_PRINT(4, ("BLOCK Mem free : allocated size = 0x%x, free size = 0x%x\n", block_mem->count * _MALI_OSK_MALI_PAGE_SIZE,
+ free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE));
+ block_mem->count = 0;
+ MALI_DEBUG_ASSERT(list_empty(&block_mem->pfns));
+
+ return free_pages_nr;
+}
+
+
+u32 mali_mem_block_free_list(struct list_head *list)
+{
+ struct mali_page_node *m_page, *m_tmp;
+ mali_block_allocator *info = mali_mem_block_gobal_allocator;
+ u32 free_pages_nr = 0;
+
+ if (info) {
+ spin_lock(&info->sp_lock);
+ list_for_each_entry_safe(m_page, m_tmp , list, list) {
+ if (1 == _mali_page_node_get_ref_count(m_page)) {
+ free_pages_nr++;
+ }
+ mali_mem_block_free_node(m_page);
+ }
+ spin_unlock(&info->sp_lock);
+ }
+ return free_pages_nr;
+}
+
+/* free the node,*/
+void mali_mem_block_free_node(struct mali_page_node *node)
+{
+ mali_block_allocator *info = mali_mem_block_gobal_allocator;
+
+ /* only handle BLOCK node */
+ if (node->type == MALI_PAGE_NODE_BLOCK && info) {
+ /*Need to make this atomic?*/
+ if (1 == _mali_page_node_get_ref_count(node)) {
+ /*Move to free list*/
+ _mali_page_node_unref(node);
+ list_move_tail(&node->list, &info->free);
+ atomic_add(1, &info->free_num);
+ } else {
+ _mali_page_node_unref(node);
+ list_del(&node->list);
+ kfree(node);
+ }
+ }
+}
+
+/* unref the node, but not free it */
+_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node)
+{
+ mali_block_allocator *info = mali_mem_block_gobal_allocator;
+ mali_page_node *new_node;
+
+ /* only handle BLOCK node */
+ if (node->type == MALI_PAGE_NODE_BLOCK && info) {
+ /*Need to make this atomic?*/
+ if (1 == _mali_page_node_get_ref_count(node)) {
+ /* allocate a new node, Add to free list, keep the old node*/
+ _mali_page_node_unref(node);
+ new_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK);
+ if (new_node) {
+ memcpy(new_node, node, sizeof(mali_page_node));
+ list_add(&new_node->list, &info->free);
+ atomic_add(1, &info->free_num);
+ } else
+ return _MALI_OSK_ERR_FAULT;
+
+ } else {
+ _mali_page_node_unref(node);
+ }
+ }
+ return _MALI_OSK_ERR_OK;
+}
+
+
+int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props)
+{
+ struct mali_page_directory *pagedir = session->page_directory;
+ struct mali_page_node *m_page;
+ dma_addr_t phys;
+ u32 virt = vaddr;
+ u32 prop = props;
+
+ list_for_each_entry(m_page, &block_mem->pfns, list) {
+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK);
+ phys = _mali_page_node_get_dma_addr(m_page);
+#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT)
+ /* Verify that the "physical" address is 32-bit and
+ * usable for Mali, when on a system with bus addresses
+ * wider than 32-bit. */
+ MALI_DEBUG_ASSERT(0 == (phys >> 32));
+#endif
+ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop);
+ virt += MALI_MMU_PAGE_SIZE;
+ }
+
+ return 0;
+}
+
+void mali_mem_block_mali_unmap(mali_mem_allocation *alloc)
+{
+ struct mali_session_data *session;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+ session = alloc->session;
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ mali_session_memory_lock(session);
+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start,
+ alloc->flags);
+ mali_session_memory_unlock(session);
+}
+
+
+int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma)
+{
+ int ret;
+ mali_mem_block_mem *block_mem = &mem_bkend->block_mem;
+ unsigned long addr = vma->vm_start;
+ struct mali_page_node *m_page;
+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK);
+
+ list_for_each_entry(m_page, &block_mem->pfns, list) {
+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK);
+ ret = vm_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page));
+
+ if (unlikely(0 != ret)) {
+ return -EFAULT;
+ }
+ addr += _MALI_OSK_MALI_PAGE_SIZE;
+
+ }
+
+ return 0;
+}
+
+
+_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size)
+{
+ mali_block_allocator *allocator;
+
+ /* Do the low level linux operation first */
+
+ /* Request ownership of the memory */
+ if (_MALI_OSK_ERR_OK != _mali_osk_mem_reqregion(start, size, "Dedicated Mali GPU memory")) {
+ MALI_DEBUG_PRINT(1, ("Failed to request memory region for frame buffer (0x%08X - 0x%08X)\n", start, start + size - 1));
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ /* Create generic block allocator object to handle it */
+ allocator = mali_mem_block_allocator_create(start, size);
+
+ if (NULL == allocator) {
+ MALI_DEBUG_PRINT(1, ("Memory bank registration failed\n"));
+ _mali_osk_mem_unreqregion(start, size);
+ MALI_ERROR(_MALI_OSK_ERR_FAULT);
+ }
+
+ mali_mem_block_gobal_allocator = (mali_block_allocator *)allocator;
+
+ return _MALI_OSK_ERR_OK;
+}
+
+mali_bool mali_memory_have_dedicated_memory(void)
+{
+ return mali_mem_block_gobal_allocator ? MALI_TRUE : MALI_FALSE;
+}
+
+u32 mali_mem_block_allocator_stat(void)
+{
+ mali_block_allocator *allocator = mali_mem_block_gobal_allocator;
+ MALI_DEBUG_ASSERT_POINTER(allocator);
+
+ return (allocator->total_num - atomic_read(&allocator->free_num)) * _MALI_OSK_MALI_PAGE_SIZE;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.h b/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.h
new file mode 100644
index 000000000000..129434de67df
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2010, 2013, 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_BLOCK_ALLOCATOR_H__
+#define __MALI_BLOCK_ALLOCATOR_H__
+
+#include "mali_session.h"
+#include "mali_memory.h"
+#include <linux/spinlock.h>
+
+#include "mali_memory_types.h"
+
+#define MALI_BLOCK_SIZE (PAGE_SIZE) /* 4 kB, manage BLOCK memory as page size */
+#define MALI_BLOCK_REF_MASK (0xFFF)
+#define MALI_BLOCK_MAX_REF_COUNT (0xFFF)
+
+
+
+typedef struct mali_block_allocator {
+ /*
+ * In free list, each node's ref_count is 0,
+ * ref_count added when allocated or referenced in COW
+ */
+ mali_block_item *items; /* information for each block item*/
+ struct list_head free; /*free list of mali_memory_node*/
+ spinlock_t sp_lock; /*lock for reference count & free list opertion*/
+ u32 total_num; /* Number of total pages*/
+ atomic_t free_num; /*number of free pages*/
+} mali_block_allocator;
+
+unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item);
+unsigned long _mali_blk_item_get_pfn(mali_block_item *item);
+u32 mali_mem_block_get_ref_count(mali_page_node *node);
+u32 mali_mem_block_add_ref(mali_page_node *node);
+u32 mali_mem_block_dec_ref(mali_page_node *node);
+u32 mali_mem_block_release(mali_mem_backend *mem_bkend);
+int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size);
+int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props);
+void mali_mem_block_mali_unmap(mali_mem_allocation *alloc);
+
+int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma);
+_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size);
+mali_bool mali_memory_have_dedicated_memory(void);
+u32 mali_mem_block_free(mali_mem_block_mem *block_mem);
+u32 mali_mem_block_free_list(struct list_head *list);
+void mali_mem_block_free_node(struct mali_page_node *node);
+void mali_mem_block_allocator_destroy(void);
+_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node);
+u32 mali_mem_block_allocator_stat(void);
+
+#endif /* __MALI_BLOCK_ALLOCATOR_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_cow.c b/drivers/gpu/arm/utgard/linux/mali_memory_cow.c
new file mode 100644
index 000000000000..bcd0f8713771
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_cow.c
@@ -0,0 +1,754 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/mm.h>
+#include <linux/list.h>
+#include <linux/mm_types.h>
+#include <linux/fs.h>
+#include <linux/dma-mapping.h>
+#include <linux/highmem.h>
+#include <asm/cacheflush.h>
+#include <linux/sched.h>
+#ifdef CONFIG_ARM
+#include <asm/outercache.h>
+#endif
+#include <asm/dma-mapping.h>
+
+#include "mali_memory.h"
+#include "mali_kernel_common.h"
+#include "mali_uk_types.h"
+#include "mali_osk.h"
+#include "mali_kernel_linux.h"
+#include "mali_memory_cow.h"
+#include "mali_memory_block_alloc.h"
+#include "mali_memory_swap_alloc.h"
+
+/**
+* allocate pages for COW backend and flush cache
+*/
+static struct page *mali_mem_cow_alloc_page(void)
+
+{
+ mali_mem_os_mem os_mem;
+ struct mali_page_node *node;
+ struct page *new_page;
+
+ int ret = 0;
+ /* allocate pages from os mem */
+ ret = mali_mem_os_alloc_pages(&os_mem, _MALI_OSK_MALI_PAGE_SIZE);
+
+ if (ret) {
+ return NULL;
+ }
+
+ MALI_DEBUG_ASSERT(1 == os_mem.count);
+
+ node = _MALI_OSK_CONTAINER_OF(os_mem.pages.next, struct mali_page_node, list);
+ new_page = node->page;
+ node->page = NULL;
+ list_del(&node->list);
+ kfree(node);
+
+ return new_page;
+}
+
+
+static struct list_head *_mali_memory_cow_get_node_list(mali_mem_backend *target_bk,
+ u32 target_offset,
+ u32 target_size)
+{
+ MALI_DEBUG_ASSERT(MALI_MEM_OS == target_bk->type || MALI_MEM_COW == target_bk->type ||
+ MALI_MEM_BLOCK == target_bk->type || MALI_MEM_SWAP == target_bk->type);
+
+ if (MALI_MEM_OS == target_bk->type) {
+ MALI_DEBUG_ASSERT(&target_bk->os_mem);
+ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->os_mem.count);
+ return &target_bk->os_mem.pages;
+ } else if (MALI_MEM_COW == target_bk->type) {
+ MALI_DEBUG_ASSERT(&target_bk->cow_mem);
+ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->cow_mem.count);
+ return &target_bk->cow_mem.pages;
+ } else if (MALI_MEM_BLOCK == target_bk->type) {
+ MALI_DEBUG_ASSERT(&target_bk->block_mem);
+ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->block_mem.count);
+ return &target_bk->block_mem.pfns;
+ } else if (MALI_MEM_SWAP == target_bk->type) {
+ MALI_DEBUG_ASSERT(&target_bk->swap_mem);
+ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->swap_mem.count);
+ return &target_bk->swap_mem.pages;
+ }
+
+ return NULL;
+}
+
+/**
+* Do COW for os memory - support do COW for memory from bank memory
+* The range_start/size can be zero, which means it will call cow_modify_range
+* latter.
+* This function allocate new pages for COW backend from os mem for a modified range
+* It will keep the page which not in the modified range and Add ref to it
+*
+* @target_bk - target allocation's backend(the allocation need to do COW)
+* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align)
+* @target_size - size of target allocation to do COW (for support memory bank)
+* @backend -COW backend
+* @range_start - offset of modified range (4K align)
+* @range_size - size of modified range
+*/
+_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk,
+ u32 target_offset,
+ u32 target_size,
+ mali_mem_backend *backend,
+ u32 range_start,
+ u32 range_size)
+{
+ mali_mem_cow *cow = &backend->cow_mem;
+ struct mali_page_node *m_page, *m_tmp, *page_node;
+ int target_page = 0;
+ struct page *new_page;
+ struct list_head *pages = NULL;
+
+ pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size);
+
+ if (NULL == pages) {
+ MALI_DEBUG_ASSERT(0);
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ MALI_DEBUG_ASSERT(0 == cow->count);
+
+ INIT_LIST_HEAD(&cow->pages);
+ mutex_lock(&target_bk->mutex);
+ list_for_each_entry_safe(m_page, m_tmp, pages, list) {
+ /* add page from (target_offset,target_offset+size) to cow backend */
+ if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) &&
+ (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) {
+
+ /* allocate a new page node, alway use OS memory for COW */
+ page_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS);
+
+ if (NULL == page_node) {
+ mutex_unlock(&target_bk->mutex);
+ goto error;
+ }
+
+ INIT_LIST_HEAD(&page_node->list);
+
+ /* check if in the modified range*/
+ if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) &&
+ (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) {
+ /* need to allocate a new page */
+ /* To simplify the case, All COW memory is allocated from os memory ?*/
+ new_page = mali_mem_cow_alloc_page();
+
+ if (NULL == new_page) {
+ kfree(page_node);
+ mutex_unlock(&target_bk->mutex);
+ goto error;
+ }
+
+ _mali_page_node_add_page(page_node, new_page);
+ } else {
+ /*Add Block memory case*/
+ if (m_page->type != MALI_PAGE_NODE_BLOCK) {
+ _mali_page_node_add_page(page_node, m_page->page);
+ } else {
+ page_node->type = MALI_PAGE_NODE_BLOCK;
+ _mali_page_node_add_block_item(page_node, m_page->blk_it);
+ }
+
+ /* add ref to this page */
+ _mali_page_node_ref(m_page);
+ }
+
+ /* add it to COW backend page list */
+ list_add_tail(&page_node->list, &cow->pages);
+ cow->count++;
+ }
+ target_page++;
+ }
+ mutex_unlock(&target_bk->mutex);
+ return _MALI_OSK_ERR_OK;
+error:
+ mali_mem_cow_release(backend, MALI_FALSE);
+ return _MALI_OSK_ERR_FAULT;
+}
+
+_mali_osk_errcode_t mali_memory_cow_swap_memory(mali_mem_backend *target_bk,
+ u32 target_offset,
+ u32 target_size,
+ mali_mem_backend *backend,
+ u32 range_start,
+ u32 range_size)
+{
+ mali_mem_cow *cow = &backend->cow_mem;
+ struct mali_page_node *m_page, *m_tmp, *page_node;
+ int target_page = 0;
+ struct mali_swap_item *swap_item;
+ struct list_head *pages = NULL;
+
+ pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size);
+ if (NULL == pages) {
+ MALI_DEBUG_ASSERT(0);
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ MALI_DEBUG_ASSERT(0 == cow->count);
+
+ INIT_LIST_HEAD(&cow->pages);
+ mutex_lock(&target_bk->mutex);
+
+ backend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN;
+
+ list_for_each_entry_safe(m_page, m_tmp, pages, list) {
+ /* add page from (target_offset,target_offset+size) to cow backend */
+ if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) &&
+ (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) {
+
+ /* allocate a new page node, use swap memory for COW memory swap cowed flag. */
+ page_node = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP);
+
+ if (NULL == page_node) {
+ mutex_unlock(&target_bk->mutex);
+ goto error;
+ }
+
+ /* check if in the modified range*/
+ if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) &&
+ (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) {
+ /* need to allocate a new page */
+ /* To simplify the case, All COW memory is allocated from os memory ?*/
+ swap_item = mali_mem_swap_alloc_swap_item();
+
+ if (NULL == swap_item) {
+ kfree(page_node);
+ mutex_unlock(&target_bk->mutex);
+ goto error;
+ }
+
+ swap_item->idx = mali_mem_swap_idx_alloc();
+
+ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) {
+ MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW.\n"));
+ kfree(page_node);
+ kfree(swap_item);
+ mutex_unlock(&target_bk->mutex);
+ goto error;
+ }
+
+ _mali_page_node_add_swap_item(page_node, swap_item);
+ } else {
+ _mali_page_node_add_swap_item(page_node, m_page->swap_it);
+
+ /* add ref to this page */
+ _mali_page_node_ref(m_page);
+ }
+
+ list_add_tail(&page_node->list, &cow->pages);
+ cow->count++;
+ }
+ target_page++;
+ }
+ mutex_unlock(&target_bk->mutex);
+
+ return _MALI_OSK_ERR_OK;
+error:
+ mali_mem_swap_release(backend, MALI_FALSE);
+ return _MALI_OSK_ERR_FAULT;
+
+}
+
+
+_mali_osk_errcode_t _mali_mem_put_page_node(mali_page_node *node)
+{
+ if (node->type == MALI_PAGE_NODE_OS) {
+ return mali_mem_os_put_page(node->page);
+ } else if (node->type == MALI_PAGE_NODE_BLOCK) {
+ return mali_mem_block_unref_node(node);
+ } else if (node->type == MALI_PAGE_NODE_SWAP) {
+ return _mali_mem_swap_put_page_node(node);
+ } else
+ MALI_DEBUG_ASSERT(0);
+ return _MALI_OSK_ERR_FAULT;
+}
+
+
+/**
+* Modify a range of a exist COW backend
+* @backend -COW backend
+* @range_start - offset of modified range (4K align)
+* @range_size - size of modified range(in byte)
+*/
+_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend,
+ u32 range_start,
+ u32 range_size)
+{
+ mali_mem_allocation *alloc = NULL;
+ mali_mem_cow *cow = &backend->cow_mem;
+ struct mali_page_node *m_page, *m_tmp;
+ LIST_HEAD(pages);
+ struct page *new_page;
+ u32 count = 0;
+ s32 change_pages_nr = 0;
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK;
+
+ if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS);
+ if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS);
+
+ alloc = backend->mali_allocation;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+
+ MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type);
+ MALI_DEBUG_ASSERT(((range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE) <= cow->count);
+
+ mutex_lock(&backend->mutex);
+
+ /* free pages*/
+ list_for_each_entry_safe(m_page, m_tmp, &cow->pages, list) {
+
+ /* check if in the modified range*/
+ if ((count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) &&
+ (count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) {
+ if (MALI_PAGE_NODE_SWAP != m_page->type) {
+ new_page = mali_mem_cow_alloc_page();
+
+ if (NULL == new_page) {
+ goto error;
+ }
+ if (1 != _mali_page_node_get_ref_count(m_page))
+ change_pages_nr++;
+ /* unref old page*/
+ if (_mali_mem_put_page_node(m_page)) {
+ __free_page(new_page);
+ goto error;
+ }
+ /* add new page*/
+ /* always use OS for COW*/
+ m_page->type = MALI_PAGE_NODE_OS;
+ _mali_page_node_add_page(m_page, new_page);
+ } else {
+ struct mali_swap_item *swap_item;
+
+ swap_item = mali_mem_swap_alloc_swap_item();
+
+ if (NULL == swap_item) {
+ goto error;
+ }
+
+ swap_item->idx = mali_mem_swap_idx_alloc();
+
+ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) {
+ MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW modify range.\n"));
+ kfree(swap_item);
+ goto error;
+ }
+
+ if (1 != _mali_page_node_get_ref_count(m_page)) {
+ change_pages_nr++;
+ }
+
+ if (_mali_mem_put_page_node(m_page)) {
+ mali_mem_swap_free_swap_item(swap_item);
+ goto error;
+ }
+
+ _mali_page_node_add_swap_item(m_page, swap_item);
+ }
+ }
+ count++;
+ }
+ cow->change_pages_nr = change_pages_nr;
+
+ MALI_DEBUG_ASSERT(MALI_MEM_COW == alloc->type);
+
+ /* ZAP cpu mapping(modified range), and do cpu mapping here if need */
+ if (NULL != alloc->cpu_mapping.vma) {
+ MALI_DEBUG_ASSERT(0 != alloc->backend_handle);
+ MALI_DEBUG_ASSERT(NULL != alloc->cpu_mapping.vma);
+ MALI_DEBUG_ASSERT(alloc->cpu_mapping.vma->vm_end - alloc->cpu_mapping.vma->vm_start >= range_size);
+
+ if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) {
+ zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size);
+
+ ret = mali_mem_cow_cpu_map_pages_locked(backend, alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size / _MALI_OSK_MALI_PAGE_SIZE);
+
+ if (unlikely(ret != _MALI_OSK_ERR_OK)) {
+ MALI_DEBUG_PRINT(2, ("mali_memory_cow_modify_range: cpu mapping failed !\n"));
+ ret = _MALI_OSK_ERR_FAULT;
+ }
+ } else {
+ /* used to trigger page fault for swappable cowed memory. */
+ alloc->cpu_mapping.vma->vm_flags |= VM_PFNMAP;
+ alloc->cpu_mapping.vma->vm_flags |= VM_MIXEDMAP;
+
+ zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size);
+ /* delete this flag to let swappble is ummapped regard to stauct page not page frame. */
+ alloc->cpu_mapping.vma->vm_flags &= ~VM_PFNMAP;
+ alloc->cpu_mapping.vma->vm_flags &= ~VM_MIXEDMAP;
+ }
+ }
+
+error:
+ mutex_unlock(&backend->mutex);
+ return ret;
+
+}
+
+
+/**
+* Allocate pages for COW backend
+* @alloc -allocation for COW allocation
+* @target_bk - target allocation's backend(the allocation need to do COW)
+* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align)
+* @target_size - size of target allocation to do COW (for support memory bank)(in byte)
+* @backend -COW backend
+* @range_start - offset of modified range (4K align)
+* @range_size - size of modified range(in byte)
+*/
+_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk,
+ u32 target_offset,
+ u32 target_size,
+ mali_mem_backend *backend,
+ u32 range_start,
+ u32 range_size)
+{
+ struct mali_session_data *session = backend->mali_allocation->session;
+
+ MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS);
+
+ /* size & offset must be a multiple of the system page size */
+ if (target_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS);
+ if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS);
+ if (target_offset % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS);
+ if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS);
+
+ /* check backend type */
+ MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type);
+
+ switch (target_bk->type) {
+ case MALI_MEM_OS:
+ case MALI_MEM_BLOCK:
+ return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size);
+ break;
+ case MALI_MEM_COW:
+ if (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) {
+ return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size);
+ } else {
+ return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size);
+ }
+ break;
+ case MALI_MEM_SWAP:
+ return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size);
+ break;
+ case MALI_MEM_EXTERNAL:
+ /*NOT support yet*/
+ MALI_DEBUG_ASSERT(0);
+ break;
+ case MALI_MEM_DMA_BUF:
+ /*NOT support yet*/
+ MALI_DEBUG_ASSERT(0);
+ break;
+ case MALI_MEM_UMP:
+ /*NOT support yet*/
+ MALI_DEBUG_ASSERT(0);
+ break;
+ default:
+ /*Not support yet*/
+ MALI_DEBUG_ASSERT(0);
+ break;
+ }
+ return _MALI_OSK_ERR_OK;
+}
+
+
+/**
+* Map COW backend memory to mali
+* Support OS/BLOCK for mali_page_node
+*/
+int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size)
+{
+ mali_mem_allocation *cow_alloc;
+ struct mali_page_node *m_page;
+ struct mali_session_data *session;
+ struct mali_page_directory *pagedir;
+ u32 virt, start;
+
+ cow_alloc = mem_bkend->mali_allocation;
+ virt = cow_alloc->mali_vma_node.vm_node.start;
+ start = virt;
+
+ MALI_DEBUG_ASSERT_POINTER(mem_bkend);
+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type);
+ MALI_DEBUG_ASSERT_POINTER(cow_alloc);
+
+ session = cow_alloc->session;
+ pagedir = session->page_directory;
+ MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS);
+ list_for_each_entry(m_page, &mem_bkend->cow_mem.pages, list) {
+ if ((virt - start >= range_start) && (virt - start < range_start + range_size)) {
+ dma_addr_t phys = _mali_page_node_get_dma_addr(m_page);
+#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT)
+ MALI_DEBUG_ASSERT(0 == (phys >> 32));
+#endif
+ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys,
+ MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT);
+ }
+ virt += MALI_MMU_PAGE_SIZE;
+ }
+ return 0;
+}
+
+/**
+* Map COW backend to cpu
+* support OS/BLOCK memory
+*/
+int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma)
+{
+ mali_mem_cow *cow = &mem_bkend->cow_mem;
+ struct mali_page_node *m_page;
+ int ret;
+ unsigned long addr = vma->vm_start;
+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW);
+
+ list_for_each_entry(m_page, &cow->pages, list) {
+ /* We should use vm_insert_page, but it does a dcache
+ * flush which makes it way slower than remap_pfn_range or vm_insert_pfn.
+ ret = vm_insert_page(vma, addr, page);
+ */
+ ret = vm_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page));
+
+ if (unlikely(0 != ret)) {
+ return ret;
+ }
+ addr += _MALI_OSK_MALI_PAGE_SIZE;
+ }
+
+ return 0;
+}
+
+/**
+* Map some pages(COW backend) to CPU vma@vaddr
+*@ mem_bkend - COW backend
+*@ vma
+*@ vaddr -start CPU vaddr mapped to
+*@ num - max number of pages to map to CPU vaddr
+*/
+_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend,
+ struct vm_area_struct *vma,
+ unsigned long vaddr,
+ int num)
+{
+ mali_mem_cow *cow = &mem_bkend->cow_mem;
+ struct mali_page_node *m_page;
+ int ret;
+ int offset;
+ int count ;
+ unsigned long vstart = vma->vm_start;
+ count = 0;
+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW);
+ MALI_DEBUG_ASSERT(0 == vaddr % _MALI_OSK_MALI_PAGE_SIZE);
+ MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE);
+ offset = (vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE;
+
+ list_for_each_entry(m_page, &cow->pages, list) {
+ if ((count >= offset) && (count < offset + num)) {
+ ret = vm_insert_pfn(vma, vaddr, _mali_page_node_get_pfn(m_page));
+
+ if (unlikely(0 != ret)) {
+ if (count == offset) {
+ return _MALI_OSK_ERR_FAULT;
+ } else {
+ /* ret is EBUSY when page isn't in modify range, but now it's OK*/
+ return _MALI_OSK_ERR_OK;
+ }
+ }
+ vaddr += _MALI_OSK_MALI_PAGE_SIZE;
+ }
+ count++;
+ }
+ return _MALI_OSK_ERR_OK;
+}
+
+/**
+* Release COW backend memory
+* free it directly(put_page--unref page), not put into pool
+*/
+u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped)
+{
+ mali_mem_allocation *alloc;
+ u32 free_pages_nr = 0;
+ MALI_DEBUG_ASSERT_POINTER(mem_bkend);
+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type);
+ alloc = mem_bkend->mali_allocation;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+
+ if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)) {
+ /* Unmap the memory from the mali virtual address space. */
+ if (MALI_TRUE == is_mali_mapped)
+ mali_mem_os_mali_unmap(alloc);
+ /* free cow backend list*/
+ free_pages_nr = mali_mem_os_free(&mem_bkend->cow_mem.pages, mem_bkend->cow_mem.count, MALI_TRUE);
+ free_pages_nr += mali_mem_block_free_list(&mem_bkend->cow_mem.pages);
+
+ MALI_DEBUG_ASSERT(list_empty(&mem_bkend->cow_mem.pages));
+ } else {
+ free_pages_nr = mali_mem_swap_release(mem_bkend, is_mali_mapped);
+ }
+
+
+ MALI_DEBUG_PRINT(4, ("COW Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->cow_mem.count * _MALI_OSK_MALI_PAGE_SIZE,
+ free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE));
+
+ mem_bkend->cow_mem.count = 0;
+ return free_pages_nr;
+}
+
+
+/* Dst node could os node or swap node. */
+void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node)
+{
+ void *dst, *src;
+ struct page *dst_page;
+ dma_addr_t dma_addr;
+
+ MALI_DEBUG_ASSERT(src_node != NULL);
+ MALI_DEBUG_ASSERT(dst_node != NULL);
+ MALI_DEBUG_ASSERT(dst_node->type == MALI_PAGE_NODE_OS
+ || dst_node->type == MALI_PAGE_NODE_SWAP);
+
+ if (dst_node->type == MALI_PAGE_NODE_OS) {
+ dst_page = dst_node->page;
+ } else {
+ dst_page = dst_node->swap_it->page;
+ }
+
+ dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(dst_node),
+ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL);
+
+ /* map it , and copy the content*/
+ dst = kmap_atomic(dst_page);
+
+ if (src_node->type == MALI_PAGE_NODE_OS ||
+ src_node->type == MALI_PAGE_NODE_SWAP) {
+ struct page *src_page;
+
+ if (src_node->type == MALI_PAGE_NODE_OS) {
+ src_page = src_node->page;
+ } else {
+ src_page = src_node->swap_it->page;
+ }
+
+ /* Clear and invaliate cache */
+ /* In ARM architecture, speculative read may pull stale data into L1 cache
+ * for kernel linear mapping page table. DMA_BIDIRECTIONAL could
+ * invalidate the L1 cache so that following read get the latest data
+ */
+ dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(src_node),
+ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL);
+
+ src = kmap_atomic(src_page);
+ memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE);
+ kunmap_atomic(src);
+ dma_addr = dma_map_page(&mali_platform_device->dev, src_page,
+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL);
+
+ if (src_node->type == MALI_PAGE_NODE_SWAP) {
+ src_node->swap_it->dma_addr = dma_addr;
+ }
+ } else if (src_node->type == MALI_PAGE_NODE_BLOCK) {
+ /*
+ * use ioremap to map src for BLOCK memory
+ */
+ src = ioremap_nocache(_mali_page_node_get_dma_addr(src_node), _MALI_OSK_MALI_PAGE_SIZE);
+ memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE);
+ iounmap(src);
+ }
+ kunmap_atomic(dst);
+ dma_addr = dma_map_page(&mali_platform_device->dev, dst_page,
+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
+
+ if (dst_node->type == MALI_PAGE_NODE_SWAP) {
+ dst_node->swap_it->dma_addr = dma_addr;
+ }
+}
+
+
+/*
+* allocate page on demand when CPU access it,
+* THis used in page fault handler
+*/
+_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page)
+{
+ struct page *new_page = NULL;
+ struct mali_page_node *new_node = NULL;
+ int i = 0;
+ struct mali_page_node *m_page, *found_node = NULL;
+ struct mali_session_data *session = NULL;
+ mali_mem_cow *cow = &mem_bkend->cow_mem;
+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type);
+ MALI_DEBUG_ASSERT(offset_page < mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE);
+ MALI_DEBUG_PRINT(4, ("mali_mem_cow_allocate_on_demand !, offset_page =0x%x\n", offset_page));
+
+ /* allocate new page here */
+ new_page = mali_mem_cow_alloc_page();
+ if (!new_page)
+ return _MALI_OSK_ERR_NOMEM;
+
+ new_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS);
+ if (!new_node) {
+ __free_page(new_page);
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ /* find the page in backend*/
+ list_for_each_entry(m_page, &cow->pages, list) {
+ if (i == offset_page) {
+ found_node = m_page;
+ break;
+ }
+ i++;
+ }
+ MALI_DEBUG_ASSERT(found_node);
+ if (NULL == found_node) {
+ __free_page(new_page);
+ kfree(new_node);
+ return _MALI_OSK_ERR_ITEM_NOT_FOUND;
+ }
+
+ _mali_page_node_add_page(new_node, new_page);
+
+ /* Copy the src page's content to new page */
+ _mali_mem_cow_copy_page(found_node, new_node);
+
+ MALI_DEBUG_ASSERT_POINTER(mem_bkend->mali_allocation);
+ session = mem_bkend->mali_allocation->session;
+ MALI_DEBUG_ASSERT_POINTER(session);
+ if (1 != _mali_page_node_get_ref_count(found_node)) {
+ atomic_add(1, &session->mali_mem_allocated_pages);
+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) {
+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE;
+ }
+ mem_bkend->cow_mem.change_pages_nr++;
+ }
+ if (_mali_mem_put_page_node(found_node)) {
+ __free_page(new_page);
+ kfree(new_node);
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ list_replace(&found_node->list, &new_node->list);
+
+ kfree(found_node);
+
+ /* map to GPU side*/
+ _mali_osk_mutex_wait(session->memory_lock);
+ mali_mem_cow_mali_map(mem_bkend, offset_page * _MALI_OSK_MALI_PAGE_SIZE, _MALI_OSK_MALI_PAGE_SIZE);
+ _mali_osk_mutex_signal(session->memory_lock);
+ return _MALI_OSK_ERR_OK;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_cow.h b/drivers/gpu/arm/utgard/linux/mali_memory_cow.h
new file mode 100644
index 000000000000..c16ec7633c0a
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_cow.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_MEMORY_COW_H__
+#define __MALI_MEMORY_COW_H__
+
+#include "mali_osk.h"
+#include "mali_session.h"
+#include "mali_memory_types.h"
+
+int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma);
+_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend,
+ struct vm_area_struct *vma,
+ unsigned long vaddr,
+ int num);
+
+_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk,
+ u32 target_offset,
+ u32 target_size,
+ mali_mem_backend *backend,
+ u32 range_start,
+ u32 range_size);
+
+_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend,
+ u32 range_start,
+ u32 range_size);
+
+_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk,
+ u32 target_offset,
+ u32 target_size,
+ mali_mem_backend *backend,
+ u32 range_start,
+ u32 range_size);
+
+void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node);
+
+int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size);
+u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped);
+_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page);
+#endif
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.c b/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.c
new file mode 100644
index 000000000000..e51902f1bf65
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.c
@@ -0,0 +1,246 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/mm.h>
+#include <linux/list.h>
+#include <linux/mm_types.h>
+#include <linux/fs.h>
+#include <linux/dma-mapping.h>
+#include <linux/highmem.h>
+#include <asm/cacheflush.h>
+#include <linux/sched.h>
+#ifdef CONFIG_ARM
+#include <asm/outercache.h>
+#endif
+#include <asm/dma-mapping.h>
+
+#include "mali_memory.h"
+#include "mali_kernel_common.h"
+#include "mali_uk_types.h"
+#include "mali_osk.h"
+#include "mali_kernel_linux.h"
+#include "mali_memory_defer_bind.h"
+#include "mali_executor.h"
+#include "mali_osk.h"
+#include "mali_scheduler.h"
+#include "mali_gp_job.h"
+
+mali_defer_bind_manager *mali_dmem_man = NULL;
+
+static u32 mali_dmem_get_gp_varying_size(struct mali_gp_job *gp_job)
+{
+ return gp_job->uargs.varying_memsize / _MALI_OSK_MALI_PAGE_SIZE;
+}
+
+_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void)
+{
+ mali_dmem_man = _mali_osk_calloc(1, sizeof(struct mali_defer_bind_manager));
+ if (!mali_dmem_man)
+ return _MALI_OSK_ERR_NOMEM;
+
+ atomic_set(&mali_dmem_man->num_used_pages, 0);
+ atomic_set(&mali_dmem_man->num_dmem, 0);
+
+ return _MALI_OSK_ERR_OK;
+}
+
+
+void mali_mem_defer_bind_manager_destory(void)
+{
+ if (mali_dmem_man) {
+ MALI_DEBUG_ASSERT(0 == atomic_read(&mali_dmem_man->num_dmem));
+ kfree(mali_dmem_man);
+ }
+ mali_dmem_man = NULL;
+}
+
+
+/*allocate pages from OS memory*/
+_mali_osk_errcode_t mali_mem_defer_alloc_mem(u32 require, struct mali_session_data *session, mali_defer_mem_block *dblock)
+{
+ int retval = 0;
+ u32 num_pages = require;
+ mali_mem_os_mem os_mem;
+
+ retval = mali_mem_os_alloc_pages(&os_mem, num_pages * _MALI_OSK_MALI_PAGE_SIZE);
+
+ /* add to free pages list */
+ if (0 == retval) {
+ MALI_DEBUG_PRINT(4, ("mali_mem_defer_alloc_mem ,,*** pages allocate = 0x%x \n", num_pages));
+ list_splice(&os_mem.pages, &dblock->free_pages);
+ atomic_add(os_mem.count, &dblock->num_free_pages);
+ atomic_add(os_mem.count, &session->mali_mem_allocated_pages);
+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) {
+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE;
+ }
+ return _MALI_OSK_ERR_OK;
+ } else
+ return _MALI_OSK_ERR_FAULT;
+}
+
+_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock)
+{
+ u32 require_page;
+
+ if (!next_gp_job)
+ return _MALI_OSK_ERR_FAULT;
+
+ require_page = mali_dmem_get_gp_varying_size(next_gp_job);
+
+ MALI_DEBUG_PRINT(4, ("mali_mem_defer_prepare_mem_work, require alloc page 0x%x\n",
+ require_page));
+ /* allocate more pages from OS */
+ if (_MALI_OSK_ERR_OK != mali_mem_defer_alloc_mem(require_page, next_gp_job->session, dblock)) {
+ MALI_DEBUG_PRINT(1, ("ERROR##mali_mem_defer_prepare_mem_work, allocate page failed!!"));
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ next_gp_job->bind_flag = MALI_DEFER_BIND_MEMORY_PREPARED;
+
+ return _MALI_OSK_ERR_OK;
+}
+
+
+/* do preparetion for allocation before defer bind */
+_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list)
+{
+ mali_mem_backend *mem_bkend = NULL;
+ struct mali_backend_bind_list *bk_list = _mali_osk_calloc(1, sizeof(struct mali_backend_bind_list));
+ if (NULL == bk_list)
+ return _MALI_OSK_ERR_FAULT;
+
+ INIT_LIST_HEAD(&bk_list->node);
+ /* Get backend memory */
+ mutex_lock(&mali_idr_mutex);
+ if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) {
+ MALI_DEBUG_PRINT(1, ("Can't find memory backend in defer bind!\n"));
+ mutex_unlock(&mali_idr_mutex);
+ kfree(bk_list);
+ return _MALI_OSK_ERR_FAULT;
+ }
+ mutex_unlock(&mali_idr_mutex);
+ MALI_DEBUG_PRINT(4, ("bind_allocation_prepare:: allocation =%x vaddr=0x%x!\n", alloc, alloc->mali_vma_node.vm_node.start));
+
+ INIT_LIST_HEAD(&mem_bkend->os_mem.pages);
+
+ bk_list->bkend = mem_bkend;
+ bk_list->vaddr = alloc->mali_vma_node.vm_node.start;
+ bk_list->session = alloc->session;
+ bk_list->page_num = mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE;
+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS);
+
+ /* add to job to do list */
+ list_add(&bk_list->node, list);
+
+ return _MALI_OSK_ERR_OK;
+}
+
+
+
+/* bind phyiscal memory to allocation
+This function will be called in IRQ handler*/
+static _mali_osk_errcode_t mali_mem_defer_bind_allocation(struct mali_backend_bind_list *bk_node,
+ struct list_head *pages)
+{
+ struct mali_session_data *session = bk_node->session;
+ mali_mem_backend *mem_bkend = bk_node->bkend;
+ MALI_DEBUG_PRINT(4, ("mali_mem_defer_bind_allocation, bind bkend = %x page num=0x%x vaddr=%x session=%x\n", mem_bkend, bk_node->page_num, bk_node->vaddr, session));
+
+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS);
+ list_splice(pages, &mem_bkend->os_mem.pages);
+ mem_bkend->os_mem.count = bk_node->page_num;
+
+ if (mem_bkend->type == MALI_MEM_OS) {
+ mali_mem_os_mali_map(&mem_bkend->os_mem, session, bk_node->vaddr, 0,
+ mem_bkend->os_mem.count, MALI_MMU_FLAGS_DEFAULT);
+ }
+ smp_wmb();
+ bk_node->flag = MALI_DEFER_BIND_MEMORY_BINDED;
+ mem_bkend->flags &= ~MALI_MEM_BACKEND_FLAG_NOT_BINDED;
+ mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_BINDED;
+ return _MALI_OSK_ERR_OK;
+}
+
+
+static struct list_head *mali_mem_defer_get_free_page_list(u32 count, struct list_head *pages, mali_defer_mem_block *dblock)
+{
+ int i = 0;
+ struct mali_page_node *m_page, *m_tmp;
+
+ if (atomic_read(&dblock->num_free_pages) < count) {
+ return NULL;
+ } else {
+ list_for_each_entry_safe(m_page, m_tmp, &dblock->free_pages, list) {
+ if (i < count) {
+ list_move_tail(&m_page->list, pages);
+ } else {
+ break;
+ }
+ i++;
+ }
+ MALI_DEBUG_ASSERT(i == count);
+ atomic_sub(count, &dblock->num_free_pages);
+ return pages;
+ }
+}
+
+
+/* called in job start IOCTL to bind physical memory for each allocations
+@ bk_list backend list to do defer bind
+@ pages page list to do this bind
+@ count number of pages
+*/
+_mali_osk_errcode_t mali_mem_defer_bind(u32 count, struct mali_gp_job *gp,
+ struct mali_defer_mem_block *dmem_block)
+{
+ struct mali_defer_mem *dmem = NULL;
+ struct mali_backend_bind_list *bkn, *bkn_tmp;
+ LIST_HEAD(pages);
+
+ MALI_DEBUG_PRINT(4, ("#BIND: GP job=%x## \n", gp));
+ dmem = (mali_defer_mem *)_mali_osk_calloc(1, sizeof(struct mali_defer_mem));
+ if (dmem) {
+ INIT_LIST_HEAD(&dmem->node);
+ gp->dmem = dmem;
+ } else {
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ atomic_add(1, &mali_dmem_man->num_dmem);
+ /* for each bk_list backend, do bind */
+ list_for_each_entry_safe(bkn, bkn_tmp , &gp->vary_todo, node) {
+ INIT_LIST_HEAD(&pages);
+ if (likely(mali_mem_defer_get_free_page_list(bkn->page_num, &pages, dmem_block))) {
+ list_del(&bkn->node);
+ mali_mem_defer_bind_allocation(bkn, &pages);
+ _mali_osk_free(bkn);
+ } else {
+ /* not enough memory will not happen */
+ MALI_DEBUG_PRINT(1, ("#BIND: NOT enough memory when binded !!## \n"));
+ MALI_DEBUG_ASSERT(0);
+ }
+ }
+
+ if (!list_empty(&gp->vary_todo)) {
+ MALI_DEBUG_ASSERT(0);
+ }
+
+ dmem->flag = MALI_DEFER_BIND_MEMORY_BINDED;
+
+ return _MALI_OSK_ERR_OK;
+}
+
+void mali_mem_defer_dmem_free(struct mali_gp_job *gp)
+{
+ if (gp->dmem) {
+ atomic_dec(&mali_dmem_man->num_dmem);
+ _mali_osk_free(gp->dmem);
+ }
+}
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.h b/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.h
new file mode 100644
index 000000000000..ef67540434f5
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.h
@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#ifndef __MALI_MEMORY_DEFER_BIND_H_
+#define __MALI_MEMORY_DEFER_BIND_H_
+
+
+#include "mali_osk.h"
+#include "mali_session.h"
+
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/rbtree.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+
+#include "mali_memory_types.h"
+#include "mali_memory_os_alloc.h"
+#include "mali_uk_types.h"
+
+struct mali_gp_job;
+
+typedef struct mali_defer_mem {
+ struct list_head node; /*dlist node in bind manager */
+ u32 flag;
+} mali_defer_mem;
+
+
+typedef struct mali_defer_mem_block {
+ struct list_head free_pages; /* page pool */
+ atomic_t num_free_pages;
+} mali_defer_mem_block;
+
+/* varying memory list need to bind */
+typedef struct mali_backend_bind_list {
+ struct list_head node;
+ struct mali_mem_backend *bkend;
+ u32 vaddr;
+ u32 page_num;
+ struct mali_session_data *session;
+ u32 flag;
+} mali_backend_bind_lists;
+
+
+typedef struct mali_defer_bind_manager {
+ atomic_t num_used_pages;
+ atomic_t num_dmem;
+} mali_defer_bind_manager;
+
+_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void);
+void mali_mem_defer_bind_manager_destory(void);
+_mali_osk_errcode_t mali_mem_defer_bind(u32 count, struct mali_gp_job *gp,
+ struct mali_defer_mem_block *dmem_block);
+_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list);
+_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock);
+void mali_mem_defer_dmem_free(struct mali_gp_job *gp);
+
+#endif
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.c b/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.c
new file mode 100644
index 000000000000..2fa5028d8342
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.c
@@ -0,0 +1,369 @@
+/*
+ * Copyright (C) 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/fs.h> /* file system operations */
+#include <asm/uaccess.h> /* user space access */
+#include <linux/dma-buf.h>
+#include <linux/scatterlist.h>
+#include <linux/rbtree.h>
+#include <linux/platform_device.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/mutex.h>
+
+#include "mali_ukk.h"
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_kernel_linux.h"
+
+#include "mali_memory.h"
+#include "mali_memory_dma_buf.h"
+#include "mali_memory_virtual.h"
+#include "mali_pp_job.h"
+
+/*
+ * Map DMA buf attachment \a mem into \a session at virtual address \a virt.
+ */
+static int mali_dma_buf_map(mali_mem_backend *mem_backend)
+{
+ mali_mem_allocation *alloc;
+ struct mali_dma_buf_attachment *mem;
+ struct mali_session_data *session;
+ struct mali_page_directory *pagedir;
+ _mali_osk_errcode_t err;
+ struct scatterlist *sg;
+ u32 virt, flags;
+ int i;
+
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+
+ alloc = mem_backend->mali_allocation;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+
+ mem = mem_backend->dma_buf.attachment;
+ MALI_DEBUG_ASSERT_POINTER(mem);
+
+ session = alloc->session;
+ MALI_DEBUG_ASSERT_POINTER(session);
+ MALI_DEBUG_ASSERT(mem->session == session);
+
+ virt = alloc->mali_vma_node.vm_node.start;
+ flags = alloc->flags;
+
+ mali_session_memory_lock(session);
+ mem->map_ref++;
+
+ MALI_DEBUG_PRINT(5, ("Mali DMA-buf: map attachment %p, new map_ref = %d\n", mem, mem->map_ref));
+
+ if (1 == mem->map_ref) {
+
+ /* First reference taken, so we need to map the dma buf */
+ MALI_DEBUG_ASSERT(!mem->is_mapped);
+
+ mem->sgt = dma_buf_map_attachment(mem->attachment, DMA_BIDIRECTIONAL);
+ if (IS_ERR_OR_NULL(mem->sgt)) {
+ MALI_DEBUG_PRINT_ERROR(("Failed to map dma-buf attachment\n"));
+ mem->map_ref--;
+ mali_session_memory_unlock(session);
+ return -EFAULT;
+ }
+
+ err = mali_mem_mali_map_prepare(alloc);
+ if (_MALI_OSK_ERR_OK != err) {
+ MALI_DEBUG_PRINT(1, ("Mapping of DMA memory failed\n"));
+ mem->map_ref--;
+ mali_session_memory_unlock(session);
+ return -ENOMEM;
+ }
+
+ pagedir = mali_session_get_page_directory(session);
+ MALI_DEBUG_ASSERT_POINTER(pagedir);
+
+ for_each_sg(mem->sgt->sgl, sg, mem->sgt->nents, i) {
+ u32 size = sg_dma_len(sg);
+ dma_addr_t phys = sg_dma_address(sg);
+
+ /* sg must be page aligned. */
+ MALI_DEBUG_ASSERT(0 == size % MALI_MMU_PAGE_SIZE);
+ MALI_DEBUG_ASSERT(0 == (phys & ~(uintptr_t)0xFFFFFFFF));
+
+ mali_mmu_pagedir_update(pagedir, virt, phys, size, MALI_MMU_FLAGS_DEFAULT);
+
+ virt += size;
+ }
+
+ if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) {
+ u32 guard_phys;
+ MALI_DEBUG_PRINT(7, ("Mapping in extra guard page\n"));
+
+ guard_phys = sg_dma_address(mem->sgt->sgl);
+ mali_mmu_pagedir_update(pagedir, virt, guard_phys, MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT);
+ }
+
+ mem->is_mapped = MALI_TRUE;
+ mali_session_memory_unlock(session);
+ /* Wake up any thread waiting for buffer to become mapped */
+ wake_up_all(&mem->wait_queue);
+ } else {
+ MALI_DEBUG_ASSERT(mem->is_mapped);
+ mali_session_memory_unlock(session);
+ }
+
+ return 0;
+}
+
+static void mali_dma_buf_unmap(mali_mem_allocation *alloc, struct mali_dma_buf_attachment *mem)
+{
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+ MALI_DEBUG_ASSERT_POINTER(mem);
+ MALI_DEBUG_ASSERT_POINTER(mem->attachment);
+ MALI_DEBUG_ASSERT_POINTER(mem->buf);
+ MALI_DEBUG_ASSERT_POINTER(alloc->session);
+
+ mali_session_memory_lock(alloc->session);
+ mem->map_ref--;
+
+ MALI_DEBUG_PRINT(5, ("Mali DMA-buf: unmap attachment %p, new map_ref = %d\n", mem, mem->map_ref));
+
+ if (0 == mem->map_ref) {
+ dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL);
+ if (MALI_TRUE == mem->is_mapped) {
+ mali_mem_mali_map_free(alloc->session, alloc->psize, alloc->mali_vma_node.vm_node.start,
+ alloc->flags);
+ }
+ mem->is_mapped = MALI_FALSE;
+ }
+ mali_session_memory_unlock(alloc->session);
+ /* Wake up any thread waiting for buffer to become unmapped */
+ wake_up_all(&mem->wait_queue);
+}
+
+#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
+int mali_dma_buf_map_job(struct mali_pp_job *job)
+{
+ struct mali_dma_buf_attachment *mem;
+ _mali_osk_errcode_t err;
+ int i;
+ int ret = 0;
+ u32 num_memory_cookies;
+ struct mali_session_data *session;
+ struct mali_vma_node *mali_vma_node = NULL;
+ mali_mem_allocation *mali_alloc = NULL;
+ mali_mem_backend *mem_bkend = NULL;
+
+ MALI_DEBUG_ASSERT_POINTER(job);
+
+ num_memory_cookies = mali_pp_job_num_memory_cookies(job);
+
+ session = mali_pp_job_get_session(job);
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ for (i = 0; i < num_memory_cookies; i++) {
+ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i);
+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
+ MALI_DEBUG_ASSERT(NULL != mali_vma_node);
+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
+ MALI_DEBUG_ASSERT(NULL != mali_alloc);
+ if (MALI_MEM_DMA_BUF != mali_alloc->type) {
+ continue;
+ }
+
+ /* Get backend memory & Map on CPU */
+ mutex_lock(&mali_idr_mutex);
+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle);
+ mutex_unlock(&mali_idr_mutex);
+ MALI_DEBUG_ASSERT(NULL != mem_bkend);
+
+ mem = mem_bkend->dma_buf.attachment;
+
+ MALI_DEBUG_ASSERT_POINTER(mem);
+ MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job));
+
+ err = mali_dma_buf_map(mem_bkend);
+ if (0 != err) {
+ MALI_DEBUG_PRINT_ERROR(("Mali DMA-buf: Failed to map dma-buf for mali address %x\n", mali_addr));
+ ret = -EFAULT;
+ continue;
+ }
+ }
+ return ret;
+}
+
+void mali_dma_buf_unmap_job(struct mali_pp_job *job)
+{
+ struct mali_dma_buf_attachment *mem;
+ int i;
+ u32 num_memory_cookies;
+ struct mali_session_data *session;
+ struct mali_vma_node *mali_vma_node = NULL;
+ mali_mem_allocation *mali_alloc = NULL;
+ mali_mem_backend *mem_bkend = NULL;
+
+ MALI_DEBUG_ASSERT_POINTER(job);
+
+ num_memory_cookies = mali_pp_job_num_memory_cookies(job);
+
+ session = mali_pp_job_get_session(job);
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ for (i = 0; i < num_memory_cookies; i++) {
+ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i);
+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
+ MALI_DEBUG_ASSERT(NULL != mali_vma_node);
+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
+ MALI_DEBUG_ASSERT(NULL != mali_alloc);
+ if (MALI_MEM_DMA_BUF != mali_alloc->type) {
+ continue;
+ }
+
+ /* Get backend memory & Map on CPU */
+ mutex_lock(&mali_idr_mutex);
+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle);
+ mutex_unlock(&mali_idr_mutex);
+ MALI_DEBUG_ASSERT(NULL != mem_bkend);
+
+ mem = mem_bkend->dma_buf.attachment;
+
+ MALI_DEBUG_ASSERT_POINTER(mem);
+ MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job));
+ mali_dma_buf_unmap(mem_bkend->mali_allocation, mem);
+ }
+}
+#endif /* !CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH */
+
+int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *user_arg)
+{
+ _mali_uk_dma_buf_get_size_s args;
+ int fd;
+ struct dma_buf *buf;
+
+ /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */
+ if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_dma_buf_get_size_s))) {
+ return -EFAULT;
+ }
+
+ /* Do DMA-BUF stuff */
+ fd = args.mem_fd;
+
+ buf = dma_buf_get(fd);
+ if (IS_ERR_OR_NULL(buf)) {
+ MALI_DEBUG_PRINT_ERROR(("Failed to get dma-buf from fd: %d\n", fd));
+ return PTR_RET(buf);
+ }
+
+ if (0 != put_user(buf->size, &user_arg->size)) {
+ dma_buf_put(buf);
+ return -EFAULT;
+ }
+
+ dma_buf_put(buf);
+
+ return 0;
+}
+
+_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc,
+ mali_mem_backend *mem_backend,
+ int fd, u32 flags)
+{
+ struct dma_buf *buf;
+ struct mali_dma_buf_attachment *dma_mem;
+ struct mali_session_data *session = alloc->session;
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+
+ /* get dma buffer */
+ buf = dma_buf_get(fd);
+ if (IS_ERR_OR_NULL(buf)) {
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ /* Currently, mapping of the full buffer are supported. */
+ if (alloc->psize != buf->size) {
+ goto failed_alloc_mem;
+ }
+
+ dma_mem = _mali_osk_calloc(1, sizeof(struct mali_dma_buf_attachment));
+ if (NULL == dma_mem) {
+ goto failed_alloc_mem;
+ }
+
+ dma_mem->buf = buf;
+ dma_mem->session = session;
+ dma_mem->map_ref = 0;
+ init_waitqueue_head(&dma_mem->wait_queue);
+
+ dma_mem->attachment = dma_buf_attach(dma_mem->buf, &mali_platform_device->dev);
+ if (NULL == dma_mem->attachment) {
+ goto failed_dma_attach;
+ }
+
+ mem_backend->dma_buf.attachment = dma_mem;
+
+ alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP;
+ if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) {
+ alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE;
+ }
+
+
+#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
+ /* Map memory into session's Mali virtual address space. */
+ if (0 != mali_dma_buf_map(mem_backend)) {
+ goto Failed_dma_map;
+ }
+#endif
+
+ return _MALI_OSK_ERR_OK;
+
+#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
+Failed_dma_map:
+ mali_dma_buf_unmap(alloc, dma_mem);
+#endif
+ /* Wait for buffer to become unmapped */
+ wait_event(dma_mem->wait_queue, !dma_mem->is_mapped);
+ MALI_DEBUG_ASSERT(!dma_mem->is_mapped);
+ dma_buf_detach(dma_mem->buf, dma_mem->attachment);
+failed_dma_attach:
+ _mali_osk_free(dma_mem);
+failed_alloc_mem:
+ dma_buf_put(buf);
+ return _MALI_OSK_ERR_FAULT;
+}
+
+void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend)
+{
+ struct mali_dma_buf_attachment *mem;
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+ MALI_DEBUG_ASSERT(MALI_MEM_DMA_BUF == mem_backend->type);
+
+ mem = mem_backend->dma_buf.attachment;
+ MALI_DEBUG_ASSERT_POINTER(mem);
+ MALI_DEBUG_ASSERT_POINTER(mem->attachment);
+ MALI_DEBUG_ASSERT_POINTER(mem->buf);
+ MALI_DEBUG_PRINT(3, ("Mali DMA-buf: release attachment %p\n", mem));
+
+#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
+ MALI_DEBUG_ASSERT_POINTER(mem_backend->mali_allocation);
+ /* We mapped implicitly on attach, so we need to unmap on release */
+ mali_dma_buf_unmap(mem_backend->mali_allocation, mem);
+#endif
+ /* Wait for buffer to become unmapped */
+ wait_event(mem->wait_queue, !mem->is_mapped);
+ MALI_DEBUG_ASSERT(!mem->is_mapped);
+
+ dma_buf_detach(mem->buf, mem->attachment);
+ dma_buf_put(mem->buf);
+
+ _mali_osk_free(mem);
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.h b/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.h
new file mode 100644
index 000000000000..859d3849e6b3
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_MEMORY_DMA_BUF_H__
+#define __MALI_MEMORY_DMA_BUF_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "mali_uk_types.h"
+#include "mali_osk.h"
+#include "mali_memory.h"
+
+struct mali_pp_job;
+
+struct mali_dma_buf_attachment;
+struct mali_dma_buf_attachment {
+ struct dma_buf *buf;
+ struct dma_buf_attachment *attachment;
+ struct sg_table *sgt;
+ struct mali_session_data *session;
+ int map_ref;
+ struct mutex map_lock;
+ mali_bool is_mapped;
+ wait_queue_head_t wait_queue;
+};
+
+int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *arg);
+
+void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend);
+
+_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc,
+ mali_mem_backend *mem_backend,
+ int fd, u32 flags);
+
+#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
+int mali_dma_buf_map_job(struct mali_pp_job *job);
+void mali_dma_buf_unmap_job(struct mali_pp_job *job);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __MALI_MEMORY_DMA_BUF_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_external.c b/drivers/gpu/arm/utgard/linux/mali_memory_external.c
new file mode 100644
index 000000000000..3733f2e0c80a
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_external.c
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include "mali_kernel_common.h"
+#include "mali_osk.h"
+#include "mali_ukk.h"
+#include "mali_memory.h"
+#include "mali_mem_validation.h"
+#include "mali_uk_types.h"
+
+void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend)
+{
+ mali_mem_allocation *alloc;
+ struct mali_session_data *session;
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+ alloc = mem_backend->mali_allocation;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+ MALI_DEBUG_ASSERT(MALI_MEM_EXTERNAL == mem_backend->type);
+
+ session = alloc->session;
+ MALI_DEBUG_ASSERT_POINTER(session);
+ mali_session_memory_lock(session);
+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start,
+ alloc->flags);
+ mali_session_memory_unlock(session);
+}
+
+_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc,
+ mali_mem_backend *mem_backend,
+ u32 phys_addr,
+ u32 flag)
+{
+ struct mali_session_data *session;
+ _mali_osk_errcode_t err;
+ u32 virt, phys, size;
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+ size = alloc->psize;
+ session = (struct mali_session_data *)(uintptr_t)alloc->session;
+ MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS);
+
+ /* check arguments */
+ /* NULL might be a valid Mali address */
+ if (!size) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS);
+
+ /* size must be a multiple of the system page size */
+ if (size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS);
+
+#if 0
+ /* Validate the mali physical range */
+ if (_MALI_OSK_ERR_OK != mali_mem_validation_check(phys_addr, size)) {
+ return _MALI_OSK_ERR_FAULT;
+ }
+#endif
+
+ if (flag & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) {
+ alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE;
+ }
+
+ mali_session_memory_lock(session);
+
+ virt = alloc->mali_vma_node.vm_node.start;
+ phys = phys_addr;
+
+ err = mali_mem_mali_map_prepare(alloc);
+ if (_MALI_OSK_ERR_OK != err) {
+ mali_session_memory_unlock(session);
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ mali_mmu_pagedir_update(session->page_directory, virt, phys, size, MALI_MMU_FLAGS_DEFAULT);
+
+ if (alloc->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) {
+ mali_mmu_pagedir_update(session->page_directory, virt + size, phys, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT);
+ }
+ MALI_DEBUG_PRINT(3,
+ ("Requested to map physical memory 0x%x-0x%x into virtual memory 0x%x\n",
+ phys_addr, (phys_addr + size - 1),
+ virt));
+ mali_session_memory_unlock(session);
+
+ MALI_SUCCESS;
+}
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_external.h b/drivers/gpu/arm/utgard/linux/mali_memory_external.h
new file mode 100644
index 000000000000..645580b51fc9
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_external.h
@@ -0,0 +1,29 @@
+
+/*
+ * Copyright (C) 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_MEMORY_EXTERNAL_H__
+#define __MALI_MEMORY_EXTERNAL_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc,
+ mali_mem_backend *mem_backend,
+ u32 phys_addr,
+ u32 flag);
+void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_manager.c b/drivers/gpu/arm/utgard/linux/mali_memory_manager.c
new file mode 100644
index 000000000000..55e2c092d597
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_manager.c
@@ -0,0 +1,965 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/fs.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/version.h>
+#include <linux/sched.h>
+
+#include <linux/platform_device.h>
+#if defined(CONFIG_DMA_SHARED_BUFFER)
+#include <linux/dma-buf.h>
+#endif
+#include <linux/idr.h>
+
+#include "mali_osk.h"
+#include "mali_osk_mali.h"
+#include "mali_kernel_linux.h"
+#include "mali_scheduler.h"
+#include "mali_memory.h"
+#include "mali_memory_os_alloc.h"
+#if defined(CONFIG_DMA_SHARED_BUFFER)
+#include "mali_memory_dma_buf.h"
+#endif
+#if defined(CONFIG_MALI400_UMP)
+#include "mali_memory_ump.h"
+#endif
+#include "mali_memory_manager.h"
+#include "mali_memory_virtual.h"
+#include "mali_memory_util.h"
+#include "mali_memory_external.h"
+#include "mali_memory_cow.h"
+#include "mali_memory_block_alloc.h"
+#include "mali_ukk.h"
+#include "mali_memory_swap_alloc.h"
+
+/*
+* New memory system interface
+*/
+
+/*inti idr for backend memory */
+struct idr mali_backend_idr;
+struct mutex mali_idr_mutex;
+
+/* init allocation manager */
+int mali_memory_manager_init(struct mali_allocation_manager *mgr)
+{
+ /* init Locks */
+ rwlock_init(&mgr->vm_lock);
+ mutex_init(&mgr->list_mutex);
+
+ /* init link */
+ INIT_LIST_HEAD(&mgr->head);
+
+ /* init RB tree */
+ mgr->allocation_mgr_rb = RB_ROOT;
+ mgr->mali_allocation_num = 0;
+ return 0;
+}
+
+/* Deinit allocation manager
+* Do some check for debug
+*/
+void mali_memory_manager_uninit(struct mali_allocation_manager *mgr)
+{
+ /* check RB tree is empty */
+ MALI_DEBUG_ASSERT(((void *)(mgr->allocation_mgr_rb.rb_node) == (void *)rb_last(&mgr->allocation_mgr_rb)));
+ /* check allocation List */
+ MALI_DEBUG_ASSERT(list_empty(&mgr->head));
+}
+
+/* Prepare memory descriptor */
+static mali_mem_allocation *mali_mem_allocation_struct_create(struct mali_session_data *session)
+{
+ mali_mem_allocation *mali_allocation;
+
+ /* Allocate memory */
+ mali_allocation = (mali_mem_allocation *)kzalloc(sizeof(mali_mem_allocation), GFP_KERNEL);
+ if (NULL == mali_allocation) {
+ MALI_DEBUG_PRINT(1, ("mali_mem_allocation_struct_create: descriptor was NULL\n"));
+ return NULL;
+ }
+
+ MALI_DEBUG_CODE(mali_allocation->magic = MALI_MEM_ALLOCATION_VALID_MAGIC);
+
+ /* do init */
+ mali_allocation->flags = 0;
+ mali_allocation->session = session;
+
+ INIT_LIST_HEAD(&mali_allocation->list);
+ _mali_osk_atomic_init(&mali_allocation->mem_alloc_refcount, 1);
+
+ /**
+ *add to session list
+ */
+ mutex_lock(&session->allocation_mgr.list_mutex);
+ list_add_tail(&mali_allocation->list, &session->allocation_mgr.head);
+ session->allocation_mgr.mali_allocation_num++;
+ mutex_unlock(&session->allocation_mgr.list_mutex);
+
+ return mali_allocation;
+}
+
+void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc)
+{
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+ MALI_DEBUG_ASSERT_POINTER(alloc->session);
+ mutex_lock(&alloc->session->allocation_mgr.list_mutex);
+ list_del(&alloc->list);
+ alloc->session->allocation_mgr.mali_allocation_num--;
+ mutex_unlock(&alloc->session->allocation_mgr.list_mutex);
+
+ kfree(alloc);
+}
+
+int mali_mem_backend_struct_create(mali_mem_backend **backend, u32 psize)
+{
+ mali_mem_backend *mem_backend = NULL;
+ s32 ret = -ENOSPC;
+ s32 index = -1;
+ *backend = (mali_mem_backend *)kzalloc(sizeof(mali_mem_backend), GFP_KERNEL);
+ if (NULL == *backend) {
+ MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: backend descriptor was NULL\n"));
+ return -1;
+ }
+ mem_backend = *backend;
+ mem_backend->size = psize;
+ mutex_init(&mem_backend->mutex);
+ INIT_LIST_HEAD(&mem_backend->list);
+ mem_backend->using_count = 0;
+
+
+ /* link backend with id */
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+again:
+ if (!idr_pre_get(&mali_backend_idr, GFP_KERNEL)) {
+ kfree(mem_backend);
+ return -ENOMEM;
+ }
+ mutex_lock(&mali_idr_mutex);
+ ret = idr_get_new_above(&mali_backend_idr, mem_backend, 1, &index);
+ mutex_unlock(&mali_idr_mutex);
+
+ if (-ENOSPC == ret) {
+ kfree(mem_backend);
+ return -ENOSPC;
+ }
+ if (-EAGAIN == ret)
+ goto again;
+#else
+ mutex_lock(&mali_idr_mutex);
+ ret = idr_alloc(&mali_backend_idr, mem_backend, 1, MALI_S32_MAX, GFP_KERNEL);
+ mutex_unlock(&mali_idr_mutex);
+ index = ret;
+ if (ret < 0) {
+ MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: Can't allocate idr for backend! \n"));
+ kfree(mem_backend);
+ return -ENOSPC;
+ }
+#endif
+ return index;
+}
+
+
+static void mali_mem_backend_struct_destory(mali_mem_backend **backend, s32 backend_handle)
+{
+ mali_mem_backend *mem_backend = *backend;
+
+ mutex_lock(&mali_idr_mutex);
+ idr_remove(&mali_backend_idr, backend_handle);
+ mutex_unlock(&mali_idr_mutex);
+ kfree(mem_backend);
+ *backend = NULL;
+}
+
+mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address)
+{
+ struct mali_vma_node *mali_vma_node = NULL;
+ mali_mem_backend *mem_bkend = NULL;
+ mali_mem_allocation *mali_alloc = NULL;
+ MALI_DEBUG_ASSERT_POINTER(session);
+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_address, 0);
+ if (NULL == mali_vma_node) {
+ MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_search:vma node was NULL\n"));
+ return NULL;
+ }
+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
+ /* Get backend memory & Map on CPU */
+ mutex_lock(&mali_idr_mutex);
+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle);
+ mutex_unlock(&mali_idr_mutex);
+ MALI_DEBUG_ASSERT(NULL != mem_bkend);
+ return mem_bkend;
+}
+
+static _mali_osk_errcode_t mali_mem_resize(struct mali_session_data *session, mali_mem_backend *mem_backend, u32 physical_size)
+{
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
+ int retval = 0;
+ mali_mem_allocation *mali_allocation = NULL;
+ mali_mem_os_mem tmp_os_mem;
+ s32 change_page_count;
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+ MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n"));
+ MALI_DEBUG_ASSERT(0 == physical_size % MALI_MMU_PAGE_SIZE);
+
+ mali_allocation = mem_backend->mali_allocation;
+ MALI_DEBUG_ASSERT_POINTER(mali_allocation);
+
+ MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE & mali_allocation->flags);
+ MALI_DEBUG_ASSERT(MALI_MEM_OS == mali_allocation->type);
+
+ mutex_lock(&mem_backend->mutex);
+
+ /* Do resize*/
+ if (physical_size > mem_backend->size) {
+ u32 add_size = physical_size - mem_backend->size;
+
+ MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE);
+
+ /* Allocate new pages from os mem */
+ retval = mali_mem_os_alloc_pages(&tmp_os_mem, add_size);
+
+ if (retval) {
+ if (-ENOMEM == retval) {
+ ret = _MALI_OSK_ERR_NOMEM;
+ } else {
+ ret = _MALI_OSK_ERR_FAULT;
+ }
+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory allocation failed !\n"));
+ goto failed_alloc_memory;
+ }
+
+ MALI_DEBUG_ASSERT(tmp_os_mem.count == add_size / MALI_MMU_PAGE_SIZE);
+
+ /* Resize the memory of the backend */
+ ret = mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count);
+
+ if (ret) {
+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory resizing failed !\n"));
+ goto failed_resize_pages;
+ }
+
+ /*Resize cpu mapping */
+ if (NULL != mali_allocation->cpu_mapping.vma) {
+ ret = mali_mem_os_resize_cpu_map_locked(mem_backend, mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + mem_backend->size, add_size);
+ if (unlikely(ret != _MALI_OSK_ERR_OK)) {
+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: cpu mapping failed !\n"));
+ goto failed_cpu_map;
+ }
+ }
+
+ /* Resize mali mapping */
+ _mali_osk_mutex_wait(session->memory_lock);
+ ret = mali_mem_mali_map_resize(mali_allocation, physical_size);
+
+ if (ret) {
+ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_resize: mali map resize fail !\n"));
+ goto failed_gpu_map;
+ }
+
+ ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, mali_allocation->mali_vma_node.vm_node.start,
+ mali_allocation->psize / MALI_MMU_PAGE_SIZE, add_size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties);
+ if (ret) {
+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: mali mapping failed !\n"));
+ goto failed_gpu_map;
+ }
+
+ _mali_osk_mutex_signal(session->memory_lock);
+ } else {
+ u32 dec_size, page_count;
+ u32 vaddr = 0;
+ INIT_LIST_HEAD(&tmp_os_mem.pages);
+ tmp_os_mem.count = 0;
+
+ dec_size = mem_backend->size - physical_size;
+ MALI_DEBUG_ASSERT(0 == dec_size % MALI_MMU_PAGE_SIZE);
+
+ page_count = dec_size / MALI_MMU_PAGE_SIZE;
+ vaddr = mali_allocation->mali_vma_node.vm_node.start + physical_size;
+
+ /* Resize the memory of the backend */
+ ret = mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, physical_size / MALI_MMU_PAGE_SIZE, page_count);
+
+ if (ret) {
+ MALI_DEBUG_PRINT(4, ("_mali_ukk_mem_resize: mali map resize failed!\n"));
+ goto failed_resize_pages;
+ }
+
+ /* Resize mali map */
+ _mali_osk_mutex_wait(session->memory_lock);
+ mali_mem_mali_map_free(session, dec_size, vaddr, mali_allocation->flags);
+ _mali_osk_mutex_signal(session->memory_lock);
+
+ /* Zap cpu mapping */
+ if (0 != mali_allocation->cpu_mapping.addr) {
+ MALI_DEBUG_ASSERT(NULL != mali_allocation->cpu_mapping.vma);
+ zap_vma_ptes(mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + physical_size, dec_size);
+ }
+
+ /* Free those extra pages */
+ mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE);
+ }
+
+ /* Resize memory allocation and memory backend */
+ change_page_count = (s32)(physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE;
+ mali_allocation->psize = physical_size;
+ mem_backend->size = physical_size;
+ mutex_unlock(&mem_backend->mutex);
+
+ if (change_page_count > 0) {
+ atomic_add(change_page_count, &session->mali_mem_allocated_pages);
+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) {
+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE;
+ }
+
+ } else {
+ atomic_sub((s32)(-change_page_count), &session->mali_mem_allocated_pages);
+ }
+
+ return _MALI_OSK_ERR_OK;
+
+failed_gpu_map:
+ _mali_osk_mutex_signal(session->memory_lock);
+failed_cpu_map:
+ if (physical_size > mem_backend->size) {
+ mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, mem_backend->size / MALI_MMU_PAGE_SIZE,
+ (physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE);
+ } else {
+ mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count);
+ }
+failed_resize_pages:
+ if (0 != tmp_os_mem.count)
+ mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE);
+failed_alloc_memory:
+
+ mutex_unlock(&mem_backend->mutex);
+ return ret;
+}
+
+
+/* Set GPU MMU properties */
+static void _mali_memory_gpu_map_property_set(u32 *properties, u32 flags)
+{
+ if (_MALI_MEMORY_GPU_READ_ALLOCATE & flags) {
+ *properties = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE;
+ } else {
+ *properties = MALI_MMU_FLAGS_DEFAULT;
+ }
+}
+
+_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size)
+{
+ mali_mem_backend *mem_backend = NULL;
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
+ mali_mem_allocation *mali_allocation = NULL;
+ u32 new_physical_size;
+ MALI_DEBUG_ASSERT_POINTER(session);
+ MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE);
+
+ /* Get the memory backend that need to be resize. */
+ mem_backend = mali_mem_backend_struct_search(session, mali_addr);
+
+ if (NULL == mem_backend) {
+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n"));
+ return ret;
+ }
+
+ mali_allocation = mem_backend->mali_allocation;
+
+ MALI_DEBUG_ASSERT_POINTER(mali_allocation);
+
+ new_physical_size = add_size + mem_backend->size;
+
+ if (new_physical_size > (mali_allocation->mali_vma_node.vm_node.size))
+ return ret;
+
+ MALI_DEBUG_ASSERT(new_physical_size != mem_backend->size);
+
+ ret = mali_mem_resize(session, mem_backend, new_physical_size);
+
+ return ret;
+}
+
+/**
+* function@_mali_ukk_mem_allocate - allocate mali memory
+*/
+_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args)
+{
+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
+ mali_mem_backend *mem_backend = NULL;
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
+ int retval = 0;
+ mali_mem_allocation *mali_allocation = NULL;
+ struct mali_vma_node *mali_vma_node = NULL;
+
+ MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_allocate, vaddr=0x%x, size =0x%x! \n", args->gpu_vaddr, args->psize));
+
+ /* Check if the address is allocated
+ */
+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->gpu_vaddr, 0);
+
+ if (unlikely(mali_vma_node)) {
+ MALI_DEBUG_ASSERT(0);
+ return _MALI_OSK_ERR_FAULT;
+ }
+ /**
+ *create mali memory allocation
+ */
+
+ mali_allocation = mali_mem_allocation_struct_create(session);
+
+ if (mali_allocation == NULL) {
+ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_allocate: Failed to create allocation struct! \n"));
+ return _MALI_OSK_ERR_NOMEM;
+ }
+ mali_allocation->psize = args->psize;
+ mali_allocation->vsize = args->vsize;
+
+ /* MALI_MEM_OS if need to support mem resize,
+ * or MALI_MEM_BLOCK if have dedicated memory,
+ * or MALI_MEM_OS,
+ * or MALI_MEM_SWAP.
+ */
+ if (args->flags & _MALI_MEMORY_ALLOCATE_SWAPPABLE) {
+ mali_allocation->type = MALI_MEM_SWAP;
+ } else if (args->flags & _MALI_MEMORY_ALLOCATE_RESIZEABLE) {
+ mali_allocation->type = MALI_MEM_OS;
+ mali_allocation->flags |= MALI_MEM_FLAG_CAN_RESIZE;
+ } else if (MALI_TRUE == mali_memory_have_dedicated_memory()) {
+ mali_allocation->type = MALI_MEM_BLOCK;
+ } else {
+ mali_allocation->type = MALI_MEM_OS;
+ }
+
+ /**
+ *add allocation node to RB tree for index
+ */
+ mali_allocation->mali_vma_node.vm_node.start = args->gpu_vaddr;
+ mali_allocation->mali_vma_node.vm_node.size = args->vsize;
+
+ mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node);
+
+ mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, args->psize);
+ if (mali_allocation->backend_handle < 0) {
+ ret = _MALI_OSK_ERR_NOMEM;
+ MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n"));
+ goto failed_alloc_backend;
+ }
+
+
+ mem_backend->mali_allocation = mali_allocation;
+ mem_backend->type = mali_allocation->type;
+
+ mali_allocation->mali_mapping.addr = args->gpu_vaddr;
+
+ /* set gpu mmu propery */
+ _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags);
+ /* do prepare for MALI mapping */
+ if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) {
+ _mali_osk_mutex_wait(session->memory_lock);
+
+ ret = mali_mem_mali_map_prepare(mali_allocation);
+ if (0 != ret) {
+ _mali_osk_mutex_signal(session->memory_lock);
+ goto failed_prepare_map;
+ }
+ _mali_osk_mutex_signal(session->memory_lock);
+ }
+
+ if (mali_allocation->psize == 0) {
+ mem_backend->os_mem.count = 0;
+ INIT_LIST_HEAD(&mem_backend->os_mem.pages);
+ goto done;
+ }
+
+ if (args->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) {
+ mali_allocation->flags |= _MALI_MEMORY_ALLOCATE_DEFER_BIND;
+ mem_backend->flags |= MALI_MEM_BACKEND_FLAG_NOT_BINDED;
+ /* init for defer bind backend*/
+ mem_backend->os_mem.count = 0;
+ INIT_LIST_HEAD(&mem_backend->os_mem.pages);
+
+ goto done;
+ }
+ /**
+ *allocate physical memory
+ */
+ if (likely(mali_allocation->psize > 0)) {
+
+ if (mem_backend->type == MALI_MEM_OS) {
+ retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size);
+ } else if (mem_backend->type == MALI_MEM_BLOCK) {
+ /* try to allocated from BLOCK memory first, then try OS memory if failed.*/
+ if (mali_mem_block_alloc(&mem_backend->block_mem, mem_backend->size)) {
+ retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size);
+ mem_backend->type = MALI_MEM_OS;
+ mali_allocation->type = MALI_MEM_OS;
+ }
+ } else if (MALI_MEM_SWAP == mem_backend->type) {
+ retval = mali_mem_swap_alloc_pages(&mem_backend->swap_mem, mali_allocation->mali_vma_node.vm_node.size, &mem_backend->start_idx);
+ } else {
+ /* ONLY support mem_os type */
+ MALI_DEBUG_ASSERT(0);
+ }
+
+ if (retval) {
+ ret = _MALI_OSK_ERR_NOMEM;
+ MALI_DEBUG_PRINT(1, (" can't allocate enough pages! \n"));
+ goto failed_alloc_pages;
+ }
+ }
+
+ /**
+ *map to GPU side
+ */
+ if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) {
+ _mali_osk_mutex_wait(session->memory_lock);
+ /* Map on Mali */
+
+ if (mem_backend->type == MALI_MEM_OS) {
+ ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, args->gpu_vaddr, 0,
+ mem_backend->size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties);
+
+ } else if (mem_backend->type == MALI_MEM_BLOCK) {
+ mali_mem_block_mali_map(&mem_backend->block_mem, session, args->gpu_vaddr,
+ mali_allocation->mali_mapping.properties);
+ } else if (mem_backend->type == MALI_MEM_SWAP) {
+ ret = mali_mem_swap_mali_map(&mem_backend->swap_mem, session, args->gpu_vaddr,
+ mali_allocation->mali_mapping.properties);
+ } else { /* unsupport type */
+ MALI_DEBUG_ASSERT(0);
+ }
+
+ _mali_osk_mutex_signal(session->memory_lock);
+ }
+done:
+ if (MALI_MEM_OS == mem_backend->type) {
+ atomic_add(mem_backend->os_mem.count, &session->mali_mem_allocated_pages);
+ } else if (MALI_MEM_BLOCK == mem_backend->type) {
+ atomic_add(mem_backend->block_mem.count, &session->mali_mem_allocated_pages);
+ } else {
+ MALI_DEBUG_ASSERT(MALI_MEM_SWAP == mem_backend->type);
+ atomic_add(mem_backend->swap_mem.count, &session->mali_mem_allocated_pages);
+ atomic_add(mem_backend->swap_mem.count, &session->mali_mem_array[mem_backend->type]);
+ }
+
+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) {
+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE;
+ }
+ return _MALI_OSK_ERR_OK;
+
+failed_alloc_pages:
+ mali_mem_mali_map_free(session, mali_allocation->psize, mali_allocation->mali_vma_node.vm_node.start, mali_allocation->flags);
+failed_prepare_map:
+ mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle);
+failed_alloc_backend:
+
+ mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node);
+ mali_mem_allocation_struct_destory(mali_allocation);
+
+ return ret;
+}
+
+
+_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args)
+{
+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
+ u32 vaddr = args->gpu_vaddr;
+ mali_mem_allocation *mali_alloc = NULL;
+ struct mali_vma_node *mali_vma_node = NULL;
+
+ /* find mali allocation structure by vaddress*/
+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, vaddr, 0);
+ if (NULL == mali_vma_node) {
+ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_free: invalid addr: 0x%x\n", vaddr));
+ return _MALI_OSK_ERR_INVALID_ARGS;
+ }
+ MALI_DEBUG_ASSERT(NULL != mali_vma_node);
+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
+
+ if (mali_alloc)
+ /* check ref_count */
+ args->free_pages_nr = mali_allocation_unref(&mali_alloc);
+
+ return _MALI_OSK_ERR_OK;
+}
+
+
+/**
+* Function _mali_ukk_mem_bind -- bind a external memory to a new GPU address
+* It will allocate a new mem allocation and bind external memory to it.
+* Supported backend type are:
+* _MALI_MEMORY_BIND_BACKEND_UMP
+* _MALI_MEMORY_BIND_BACKEND_DMA_BUF
+* _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY
+* CPU access is not supported yet
+*/
+_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args)
+{
+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
+ mali_mem_backend *mem_backend = NULL;
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
+ mali_mem_allocation *mali_allocation = NULL;
+ MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_bind, vaddr=0x%x, size =0x%x! \n", args->vaddr, args->size));
+
+ /**
+ * allocate mali allocation.
+ */
+ mali_allocation = mali_mem_allocation_struct_create(session);
+
+ if (mali_allocation == NULL) {
+ return _MALI_OSK_ERR_NOMEM;
+ }
+ mali_allocation->psize = args->size;
+ mali_allocation->vsize = args->size;
+ mali_allocation->mali_mapping.addr = args->vaddr;
+
+ /* add allocation node to RB tree for index */
+ mali_allocation->mali_vma_node.vm_node.start = args->vaddr;
+ mali_allocation->mali_vma_node.vm_node.size = args->size;
+ mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node);
+
+ /* allocate backend*/
+ if (mali_allocation->psize > 0) {
+ mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize);
+ if (mali_allocation->backend_handle < 0) {
+ goto Failed_alloc_backend;
+ }
+
+ } else {
+ goto Failed_alloc_backend;
+ }
+
+ mem_backend->size = mali_allocation->psize;
+ mem_backend->mali_allocation = mali_allocation;
+
+ switch (args->flags & _MALI_MEMORY_BIND_BACKEND_MASK) {
+ case _MALI_MEMORY_BIND_BACKEND_UMP:
+#if defined(CONFIG_MALI400_UMP)
+ mali_allocation->type = MALI_MEM_UMP;
+ mem_backend->type = MALI_MEM_UMP;
+ ret = mali_mem_bind_ump_buf(mali_allocation, mem_backend,
+ args->mem_union.bind_ump.secure_id, args->mem_union.bind_ump.flags);
+ if (_MALI_OSK_ERR_OK != ret) {
+ MALI_DEBUG_PRINT(1, ("Bind ump buf failed\n"));
+ goto Failed_bind_backend;
+ }
+#else
+ MALI_DEBUG_PRINT(1, ("UMP not supported\n"));
+ goto Failed_bind_backend;
+#endif
+ break;
+ case _MALI_MEMORY_BIND_BACKEND_DMA_BUF:
+#if defined(CONFIG_DMA_SHARED_BUFFER)
+ mali_allocation->type = MALI_MEM_DMA_BUF;
+ mem_backend->type = MALI_MEM_DMA_BUF;
+ ret = mali_mem_bind_dma_buf(mali_allocation, mem_backend,
+ args->mem_union.bind_dma_buf.mem_fd, args->mem_union.bind_dma_buf.flags);
+ if (_MALI_OSK_ERR_OK != ret) {
+ MALI_DEBUG_PRINT(1, ("Bind dma buf failed\n"));
+ goto Failed_bind_backend;
+ }
+#else
+ MALI_DEBUG_PRINT(1, ("DMA not supported\n"));
+ goto Failed_bind_backend;
+#endif
+ break;
+ case _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY:
+ /* not allowed */
+ MALI_DEBUG_ASSERT(0);
+ break;
+
+ case _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY:
+ mali_allocation->type = MALI_MEM_EXTERNAL;
+ mem_backend->type = MALI_MEM_EXTERNAL;
+ ret = mali_mem_bind_ext_buf(mali_allocation, mem_backend, args->mem_union.bind_ext_memory.phys_addr,
+ args->mem_union.bind_ext_memory.flags);
+ if (_MALI_OSK_ERR_OK != ret) {
+ MALI_DEBUG_PRINT(1, ("Bind external buf failed\n"));
+ goto Failed_bind_backend;
+ }
+ break;
+
+ case _MALI_MEMORY_BIND_BACKEND_EXT_COW:
+ /* not allowed */
+ MALI_DEBUG_ASSERT(0);
+ break;
+
+ default:
+ MALI_DEBUG_ASSERT(0);
+ break;
+ }
+ MALI_DEBUG_ASSERT(0 == mem_backend->size % MALI_MMU_PAGE_SIZE);
+ atomic_add(mem_backend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_backend->type]);
+ return _MALI_OSK_ERR_OK;
+
+Failed_bind_backend:
+ mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle);
+
+Failed_alloc_backend:
+ mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node);
+ mali_mem_allocation_struct_destory(mali_allocation);
+
+ MALI_DEBUG_PRINT(1, (" _mali_ukk_mem_bind, return ERROR! \n"));
+ return ret;
+}
+
+
+/*
+* Function _mali_ukk_mem_unbind -- unbind a external memory to a new GPU address
+* This function unbind the backend memory and free the allocation
+* no ref_count for this type of memory
+*/
+_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args)
+{
+ /**/
+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
+ mali_mem_allocation *mali_allocation = NULL;
+ struct mali_vma_node *mali_vma_node = NULL;
+ u32 mali_addr = args->vaddr;
+ MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_unbind, vaddr=0x%x! \n", args->vaddr));
+
+ /* find the allocation by vaddr */
+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
+ if (likely(mali_vma_node)) {
+ MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start);
+ mali_allocation = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
+ } else {
+ MALI_DEBUG_ASSERT(NULL != mali_vma_node);
+ return _MALI_OSK_ERR_INVALID_ARGS;
+ }
+
+ if (NULL != mali_allocation)
+ /* check ref_count */
+ mali_allocation_unref(&mali_allocation);
+ return _MALI_OSK_ERR_OK;
+}
+
+/*
+* Function _mali_ukk_mem_cow -- COW for an allocation
+* This function allocate new pages for a range (range, range+size) of allocation
+* And Map it(keep use the not in range pages from target allocation ) to an GPU vaddr
+*/
+_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args)
+{
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
+ mali_mem_backend *target_backend = NULL;
+ mali_mem_backend *mem_backend = NULL;
+ struct mali_vma_node *mali_vma_node = NULL;
+ mali_mem_allocation *mali_allocation = NULL;
+
+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
+ /* Get the target backend for cow */
+ target_backend = mali_mem_backend_struct_search(session, args->target_handle);
+
+ if (NULL == target_backend || 0 == target_backend->size) {
+ MALI_DEBUG_ASSERT_POINTER(target_backend);
+ MALI_DEBUG_ASSERT(0 != target_backend->size);
+ return ret;
+ }
+
+ /*Cow not support resized mem */
+ MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE != (MALI_MEM_FLAG_CAN_RESIZE & target_backend->mali_allocation->flags));
+
+ /* Check if the new mali address is allocated */
+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->vaddr, 0);
+
+ if (unlikely(mali_vma_node)) {
+ MALI_DEBUG_ASSERT(0);
+ return ret;
+ }
+
+ /* create new alloction for COW*/
+ mali_allocation = mali_mem_allocation_struct_create(session);
+ if (mali_allocation == NULL) {
+ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to create allocation struct!\n"));
+ return _MALI_OSK_ERR_NOMEM;
+ }
+ mali_allocation->psize = args->target_size;
+ mali_allocation->vsize = args->target_size;
+ mali_allocation->type = MALI_MEM_COW;
+
+ /*add allocation node to RB tree for index*/
+ mali_allocation->mali_vma_node.vm_node.start = args->vaddr;
+ mali_allocation->mali_vma_node.vm_node.size = mali_allocation->vsize;
+ mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node);
+
+ /* create new backend for COW memory */
+ mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize);
+ if (mali_allocation->backend_handle < 0) {
+ ret = _MALI_OSK_ERR_NOMEM;
+ MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n"));
+ goto failed_alloc_backend;
+ }
+ mem_backend->mali_allocation = mali_allocation;
+ mem_backend->type = mali_allocation->type;
+
+ if (target_backend->type == MALI_MEM_SWAP ||
+ (MALI_MEM_COW == target_backend->type && (MALI_MEM_BACKEND_FLAG_SWAP_COWED & target_backend->flags))) {
+ mem_backend->flags |= MALI_MEM_BACKEND_FLAG_SWAP_COWED;
+ /**
+ * CoWed swap backends couldn't be mapped as non-linear vma, because if one
+ * vma is set with flag VM_NONLINEAR, the vma->vm_private_data will be used by kernel,
+ * while in mali driver, we use this variable to store the pointer of mali_allocation, so there
+ * is a conflict.
+ * To resolve this problem, we have to do some fake things, we reserved about 64MB
+ * space from index 0, there isn't really page's index will be set from 0 to (64MB>>PAGE_SHIFT_NUM),
+ * and all of CoWed swap memory backends' start_idx will be assigned with 0, and these
+ * backends will be mapped as linear and will add to priority tree of global swap file, while
+ * these vmas will never be found by using normal page->index, these pages in those vma
+ * also couldn't be swapped out.
+ */
+ mem_backend->start_idx = 0;
+ }
+
+ /* Add the target backend's cow count, also allocate new pages for COW backend from os mem
+ *for a modified range and keep the page which not in the modified range and Add ref to it
+ */
+ MALI_DEBUG_PRINT(3, ("Cow mapping: target_addr: 0x%x; cow_addr: 0x%x, size: %u\n", target_backend->mali_allocation->mali_vma_node.vm_node.start,
+ mali_allocation->mali_vma_node.vm_node.start, mali_allocation->mali_vma_node.vm_node.size));
+
+ ret = mali_memory_do_cow(target_backend, args->target_offset, args->target_size, mem_backend, args->range_start, args->range_size);
+ if (_MALI_OSK_ERR_OK != ret) {
+ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to cow!\n"));
+ goto failed_do_cow;
+ }
+
+ /**
+ *map to GPU side
+ */
+ mali_allocation->mali_mapping.addr = args->vaddr;
+ /* set gpu mmu propery */
+ _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags);
+
+ _mali_osk_mutex_wait(session->memory_lock);
+ /* Map on Mali */
+ ret = mali_mem_mali_map_prepare(mali_allocation);
+ if (0 != ret) {
+ MALI_DEBUG_PRINT(1, (" prepare map fail! \n"));
+ goto failed_gpu_map;
+ }
+
+ if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) {
+ mali_mem_cow_mali_map(mem_backend, 0, mem_backend->size);
+ }
+
+ _mali_osk_mutex_signal(session->memory_lock);
+
+ mutex_lock(&target_backend->mutex);
+ target_backend->flags |= MALI_MEM_BACKEND_FLAG_COWED;
+ mutex_unlock(&target_backend->mutex);
+
+ atomic_add(args->range_size / MALI_MMU_PAGE_SIZE, &session->mali_mem_allocated_pages);
+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) {
+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE;
+ }
+ return _MALI_OSK_ERR_OK;
+
+failed_gpu_map:
+ _mali_osk_mutex_signal(session->memory_lock);
+ mali_mem_cow_release(mem_backend, MALI_FALSE);
+ mem_backend->cow_mem.count = 0;
+failed_do_cow:
+ mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle);
+failed_alloc_backend:
+ mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node);
+ mali_mem_allocation_struct_destory(mali_allocation);
+
+ return ret;
+}
+
+_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args)
+{
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
+ mali_mem_backend *mem_backend = NULL;
+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
+
+ MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_cow_modify_range called! \n"));
+ /* Get the backend that need to be modified. */
+ mem_backend = mali_mem_backend_struct_search(session, args->vaddr);
+
+ if (NULL == mem_backend || 0 == mem_backend->size) {
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+ MALI_DEBUG_ASSERT(0 != mem_backend->size);
+ return ret;
+ }
+
+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_backend->type);
+
+ ret = mali_memory_cow_modify_range(mem_backend, args->range_start, args->size);
+ args->change_pages_nr = mem_backend->cow_mem.change_pages_nr;
+ if (_MALI_OSK_ERR_OK != ret)
+ return ret;
+ _mali_osk_mutex_wait(session->memory_lock);
+ if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) {
+ mali_mem_cow_mali_map(mem_backend, args->range_start, args->size);
+ }
+ _mali_osk_mutex_signal(session->memory_lock);
+
+ atomic_add(args->change_pages_nr, &session->mali_mem_allocated_pages);
+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) {
+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE;
+ }
+
+ return _MALI_OSK_ERR_OK;
+}
+
+
+_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args)
+{
+ mali_mem_backend *mem_backend = NULL;
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT;
+
+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+ MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n"));
+ MALI_DEBUG_ASSERT(0 == args->psize % MALI_MMU_PAGE_SIZE);
+
+ /* Get the memory backend that need to be resize. */
+ mem_backend = mali_mem_backend_struct_search(session, args->vaddr);
+
+ if (NULL == mem_backend) {
+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n"));
+ return ret;
+ }
+
+ MALI_DEBUG_ASSERT(args->psize != mem_backend->size);
+
+ ret = mali_mem_resize(session, mem_backend, args->psize);
+
+ return ret;
+}
+
+_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args)
+{
+ args->memory_usage = _mali_ukk_report_memory_usage();
+ if (0 != args->vaddr) {
+ mali_mem_backend *mem_backend = NULL;
+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
+ /* Get the backend that need to be modified. */
+ mem_backend = mali_mem_backend_struct_search(session, args->vaddr);
+ if (NULL == mem_backend) {
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ if (MALI_MEM_COW == mem_backend->type)
+ args->change_pages_nr = mem_backend->cow_mem.change_pages_nr;
+ }
+ return _MALI_OSK_ERR_OK;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_manager.h b/drivers/gpu/arm/utgard/linux/mali_memory_manager.h
new file mode 100644
index 000000000000..c454b9354676
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_manager.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_MEMORY_MANAGER_H__
+#define __MALI_MEMORY_MANAGER_H__
+
+#include "mali_osk.h"
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/rbtree.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+#include "mali_memory_types.h"
+#include "mali_memory_os_alloc.h"
+#include "mali_uk_types.h"
+
+struct mali_allocation_manager {
+ rwlock_t vm_lock;
+ struct rb_root allocation_mgr_rb;
+ struct list_head head;
+ struct mutex list_mutex;
+ u32 mali_allocation_num;
+};
+
+extern struct idr mali_backend_idr;
+extern struct mutex mali_idr_mutex;
+
+int mali_memory_manager_init(struct mali_allocation_manager *mgr);
+void mali_memory_manager_uninit(struct mali_allocation_manager *mgr);
+
+void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc);
+_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size);
+mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address);
+_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args);
+_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args);
+_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args);
+_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args);
+_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args);
+_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args);
+_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args);
+_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args);
+
+#endif
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.c b/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.c
new file mode 100644
index 000000000000..1a6cc0649421
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.c
@@ -0,0 +1,805 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/fs.h>
+#include <linux/dma-mapping.h>
+#include <linux/version.h>
+#include <linux/platform_device.h>
+#include <linux/workqueue.h>
+
+#include "mali_osk.h"
+#include "mali_memory.h"
+#include "mali_memory_os_alloc.h"
+#include "mali_kernel_linux.h"
+
+/* Minimum size of allocator page pool */
+#define MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB * 256)
+#define MALI_OS_MEMORY_POOL_TRIM_JIFFIES (10 * CONFIG_HZ) /* Default to 10s */
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
+/* Write combine dma_attrs */
+static DEFINE_DMA_ATTRS(dma_attrs_wc);
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
+static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask);
+#else
+static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask);
+#endif
+#else
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
+static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc);
+#else
+static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc);
+static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc);
+#endif
+#endif
+static void mali_mem_os_trim_pool(struct work_struct *work);
+
+struct mali_mem_os_allocator mali_mem_os_allocator = {
+ .pool_lock = __SPIN_LOCK_UNLOCKED(pool_lock),
+ .pool_pages = LIST_HEAD_INIT(mali_mem_os_allocator.pool_pages),
+ .pool_count = 0,
+
+ .allocated_pages = ATOMIC_INIT(0),
+ .allocation_limit = 0,
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
+ .shrinker.shrink = mali_mem_os_shrink,
+#else
+ .shrinker.count_objects = mali_mem_os_shrink_count,
+ .shrinker.scan_objects = mali_mem_os_shrink,
+#endif
+ .shrinker.seeks = DEFAULT_SEEKS,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)
+ .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool, TIMER_DEFERRABLE),
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)
+ .timed_shrinker = __DEFERRED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool),
+#else
+ .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool),
+#endif
+};
+
+u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag)
+{
+ LIST_HEAD(pages);
+ struct mali_page_node *m_page, *m_tmp;
+ u32 free_pages_nr = 0;
+
+ if (MALI_TRUE == cow_flag) {
+ list_for_each_entry_safe(m_page, m_tmp, os_pages, list) {
+ /*only handle OS node here */
+ if (m_page->type == MALI_PAGE_NODE_OS) {
+ if (1 == _mali_page_node_get_ref_count(m_page)) {
+ list_move(&m_page->list, &pages);
+ atomic_sub(1, &mali_mem_os_allocator.allocated_pages);
+ free_pages_nr ++;
+ } else {
+ _mali_page_node_unref(m_page);
+ m_page->page = NULL;
+ list_del(&m_page->list);
+ kfree(m_page);
+ }
+ }
+ }
+ } else {
+ list_cut_position(&pages, os_pages, os_pages->prev);
+ atomic_sub(pages_count, &mali_mem_os_allocator.allocated_pages);
+ free_pages_nr = pages_count;
+ }
+
+ /* Put pages on pool. */
+ spin_lock(&mali_mem_os_allocator.pool_lock);
+ list_splice(&pages, &mali_mem_os_allocator.pool_pages);
+ mali_mem_os_allocator.pool_count += free_pages_nr;
+ spin_unlock(&mali_mem_os_allocator.pool_lock);
+
+ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) {
+ MALI_DEBUG_PRINT(5, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count));
+ queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES);
+ }
+ return free_pages_nr;
+}
+
+/**
+* put page without put it into page pool
+*/
+_mali_osk_errcode_t mali_mem_os_put_page(struct page *page)
+{
+ MALI_DEBUG_ASSERT_POINTER(page);
+ if (1 == page_count(page)) {
+ atomic_sub(1, &mali_mem_os_allocator.allocated_pages);
+ dma_unmap_page(&mali_platform_device->dev, page_private(page),
+ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
+ ClearPagePrivate(page);
+ }
+ put_page(page);
+ return _MALI_OSK_ERR_OK;
+}
+
+_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count)
+{
+ struct mali_page_node *m_page, *m_tmp;
+ u32 i = 0;
+
+ MALI_DEBUG_ASSERT_POINTER(mem_from);
+ MALI_DEBUG_ASSERT_POINTER(mem_to);
+
+ if (mem_from->count < start_page + page_count) {
+ return _MALI_OSK_ERR_INVALID_ARGS;
+ }
+
+ list_for_each_entry_safe(m_page, m_tmp, &mem_from->pages, list) {
+ if (i >= start_page && i < start_page + page_count) {
+ list_move_tail(&m_page->list, &mem_to->pages);
+ mem_from->count--;
+ mem_to->count++;
+ }
+ i++;
+ }
+
+ return _MALI_OSK_ERR_OK;
+}
+
+
+int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size)
+{
+ struct page *new_page;
+ LIST_HEAD(pages_list);
+ size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE;
+ size_t remaining = page_count;
+ struct mali_page_node *m_page, *m_tmp;
+ u32 i;
+
+ MALI_DEBUG_ASSERT_POINTER(os_mem);
+
+ if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) {
+ MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n",
+ size,
+ atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE,
+ mali_mem_os_allocator.allocation_limit));
+ return -ENOMEM;
+ }
+
+ INIT_LIST_HEAD(&os_mem->pages);
+ os_mem->count = page_count;
+
+ /* Grab pages from pool. */
+ {
+ size_t pool_pages;
+ spin_lock(&mali_mem_os_allocator.pool_lock);
+ pool_pages = min(remaining, mali_mem_os_allocator.pool_count);
+ for (i = pool_pages; i > 0; i--) {
+ BUG_ON(list_empty(&mali_mem_os_allocator.pool_pages));
+ list_move(mali_mem_os_allocator.pool_pages.next, &pages_list);
+ }
+ mali_mem_os_allocator.pool_count -= pool_pages;
+ remaining -= pool_pages;
+ spin_unlock(&mali_mem_os_allocator.pool_lock);
+ }
+
+ /* Process pages from pool. */
+ i = 0;
+ list_for_each_entry_safe(m_page, m_tmp, &pages_list, list) {
+ BUG_ON(NULL == m_page);
+
+ list_move_tail(&m_page->list, &os_mem->pages);
+ }
+
+ /* Allocate new pages, if needed. */
+ for (i = 0; i < remaining; i++) {
+ dma_addr_t dma_addr;
+ gfp_t flags = __GFP_ZERO | __GFP_NORETRY | __GFP_NOWARN | __GFP_COLD;
+ int err;
+
+#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE)
+ flags |= GFP_HIGHUSER;
+#else
+#ifdef CONFIG_ZONE_DMA32
+ flags |= GFP_DMA32;
+#else
+#ifdef CONFIG_ZONE_DMA
+ flags |= GFP_DMA;
+#else
+ /* arm64 utgard only work on < 4G, but the kernel
+ * didn't provide method to allocte memory < 4G
+ */
+ MALI_DEBUG_ASSERT(0);
+#endif
+#endif
+#endif
+
+ new_page = alloc_page(flags);
+
+ if (unlikely(NULL == new_page)) {
+ /* Calculate the number of pages actually allocated, and free them. */
+ os_mem->count = (page_count - remaining) + i;
+ atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages);
+ mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE);
+ return -ENOMEM;
+ }
+
+ /* Ensure page is flushed from CPU caches. */
+ dma_addr = dma_map_page(&mali_platform_device->dev, new_page,
+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
+
+ err = dma_mapping_error(&mali_platform_device->dev, dma_addr);
+ if (unlikely(err)) {
+ MALI_DEBUG_PRINT_ERROR(("OS Mem: Failed to DMA map page %p: %u",
+ new_page, err));
+ __free_page(new_page);
+ os_mem->count = (page_count - remaining) + i;
+ atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages);
+ mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE);
+ return -EFAULT;
+ }
+
+ /* Store page phys addr */
+ SetPagePrivate(new_page);
+ set_page_private(new_page, dma_addr);
+
+ m_page = _mali_page_node_allocate(MALI_PAGE_NODE_OS);
+ if (unlikely(NULL == m_page)) {
+ MALI_PRINT_ERROR(("OS Mem: Can't allocate mali_page node! \n"));
+ dma_unmap_page(&mali_platform_device->dev, page_private(new_page),
+ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
+ ClearPagePrivate(new_page);
+ __free_page(new_page);
+ os_mem->count = (page_count - remaining) + i;
+ atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages);
+ mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE);
+ return -EFAULT;
+ }
+ m_page->page = new_page;
+
+ list_add_tail(&m_page->list, &os_mem->pages);
+ }
+
+ atomic_add(page_count, &mali_mem_os_allocator.allocated_pages);
+
+ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) {
+ MALI_DEBUG_PRINT(4, ("OS Mem: Stopping pool trim timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count));
+ cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker);
+ }
+
+ return 0;
+}
+
+
+_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props)
+{
+ struct mali_page_directory *pagedir = session->page_directory;
+ struct mali_page_node *m_page;
+ u32 virt;
+ u32 prop = props;
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+ MALI_DEBUG_ASSERT_POINTER(os_mem);
+
+ MALI_DEBUG_ASSERT(start_page <= os_mem->count);
+ MALI_DEBUG_ASSERT((start_page + mapping_pgae_num) <= os_mem->count);
+
+ if ((start_page + mapping_pgae_num) == os_mem->count) {
+
+ virt = vaddr + MALI_MMU_PAGE_SIZE * (start_page + mapping_pgae_num);
+
+ list_for_each_entry_reverse(m_page, &os_mem->pages, list) {
+
+ virt -= MALI_MMU_PAGE_SIZE;
+ if (mapping_pgae_num > 0) {
+ dma_addr_t phys = page_private(m_page->page);
+#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT)
+ /* Verify that the "physical" address is 32-bit and
+ * usable for Mali, when on a system with bus addresses
+ * wider than 32-bit. */
+ MALI_DEBUG_ASSERT(0 == (phys >> 32));
+#endif
+ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop);
+ } else {
+ break;
+ }
+ mapping_pgae_num--;
+ }
+
+ } else {
+ u32 i = 0;
+ virt = vaddr;
+ list_for_each_entry(m_page, &os_mem->pages, list) {
+
+ if (i >= start_page) {
+ dma_addr_t phys = page_private(m_page->page);
+
+#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT)
+ /* Verify that the "physical" address is 32-bit and
+ * usable for Mali, when on a system with bus addresses
+ * wider than 32-bit. */
+ MALI_DEBUG_ASSERT(0 == (phys >> 32));
+#endif
+ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop);
+ }
+ i++;
+ virt += MALI_MMU_PAGE_SIZE;
+ }
+ }
+ return _MALI_OSK_ERR_OK;
+}
+
+
+void mali_mem_os_mali_unmap(mali_mem_allocation *alloc)
+{
+ struct mali_session_data *session;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+ session = alloc->session;
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ mali_session_memory_lock(session);
+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start,
+ alloc->flags);
+ mali_session_memory_unlock(session);
+}
+
+int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma)
+{
+ mali_mem_os_mem *os_mem = &mem_bkend->os_mem;
+ struct mali_page_node *m_page;
+ struct page *page;
+ int ret;
+ unsigned long addr = vma->vm_start;
+ MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type);
+
+ list_for_each_entry(m_page, &os_mem->pages, list) {
+ /* We should use vm_insert_page, but it does a dcache
+ * flush which makes it way slower than remap_pfn_range or vm_insert_pfn.
+ ret = vm_insert_page(vma, addr, page);
+ */
+ page = m_page->page;
+ ret = vm_insert_pfn(vma, addr, page_to_pfn(page));
+
+ if (unlikely(0 != ret)) {
+ return -EFAULT;
+ }
+ addr += _MALI_OSK_MALI_PAGE_SIZE;
+ }
+
+ return 0;
+}
+
+_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size)
+{
+ mali_mem_os_mem *os_mem = &mem_bkend->os_mem;
+ struct mali_page_node *m_page;
+ int ret;
+ int offset;
+ int mapping_page_num;
+ int count ;
+
+ unsigned long vstart = vma->vm_start;
+ count = 0;
+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS);
+ MALI_DEBUG_ASSERT(0 == start_vaddr % _MALI_OSK_MALI_PAGE_SIZE);
+ MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE);
+ offset = (start_vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE;
+ MALI_DEBUG_ASSERT(offset <= os_mem->count);
+ mapping_page_num = mappig_size / _MALI_OSK_MALI_PAGE_SIZE;
+ MALI_DEBUG_ASSERT((offset + mapping_page_num) <= os_mem->count);
+
+ if ((offset + mapping_page_num) == os_mem->count) {
+
+ unsigned long vm_end = start_vaddr + mappig_size;
+
+ list_for_each_entry_reverse(m_page, &os_mem->pages, list) {
+
+ vm_end -= _MALI_OSK_MALI_PAGE_SIZE;
+ if (mapping_page_num > 0) {
+ ret = vm_insert_pfn(vma, vm_end, page_to_pfn(m_page->page));
+
+ if (unlikely(0 != ret)) {
+ /*will return -EBUSY If the page has already been mapped into table, but it's OK*/
+ if (-EBUSY == ret) {
+ break;
+ } else {
+ MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, offset is %d,page_count is %d\n",
+ ret, offset + mapping_page_num, os_mem->count));
+ }
+ return _MALI_OSK_ERR_FAULT;
+ }
+ } else {
+ break;
+ }
+ mapping_page_num--;
+
+ }
+ } else {
+
+ list_for_each_entry(m_page, &os_mem->pages, list) {
+ if (count >= offset) {
+
+ ret = vm_insert_pfn(vma, vstart, page_to_pfn(m_page->page));
+
+ if (unlikely(0 != ret)) {
+ /*will return -EBUSY If the page has already been mapped into table, but it's OK*/
+ if (-EBUSY == ret) {
+ break;
+ } else {
+ MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, count is %d, offset is %d,page_count is %d\n",
+ ret, count, offset, os_mem->count));
+ }
+ return _MALI_OSK_ERR_FAULT;
+ }
+ }
+ count++;
+ vstart += _MALI_OSK_MALI_PAGE_SIZE;
+ }
+ }
+ return _MALI_OSK_ERR_OK;
+}
+
+u32 mali_mem_os_release(mali_mem_backend *mem_bkend)
+{
+
+ mali_mem_allocation *alloc;
+ u32 free_pages_nr = 0;
+ MALI_DEBUG_ASSERT_POINTER(mem_bkend);
+ MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type);
+
+ alloc = mem_bkend->mali_allocation;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+
+ /* Unmap the memory from the mali virtual address space. */
+ mali_mem_os_mali_unmap(alloc);
+ mutex_lock(&mem_bkend->mutex);
+ /* Free pages */
+ if (MALI_MEM_BACKEND_FLAG_COWED & mem_bkend->flags) {
+ free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_TRUE);
+ } else {
+ free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_FALSE);
+ }
+ mutex_unlock(&mem_bkend->mutex);
+
+ MALI_DEBUG_PRINT(4, ("OS Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->os_mem.count * _MALI_OSK_MALI_PAGE_SIZE,
+ free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE));
+
+ mem_bkend->os_mem.count = 0;
+ return free_pages_nr;
+}
+
+
+#define MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE 128
+static struct {
+ struct {
+ mali_dma_addr phys;
+ mali_io_address mapping;
+ } page[MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE];
+ size_t count;
+ spinlock_t lock;
+} mali_mem_page_table_page_pool = {
+ .count = 0,
+ .lock = __SPIN_LOCK_UNLOCKED(pool_lock),
+};
+
+_mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping)
+{
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_NOMEM;
+ dma_addr_t tmp_phys;
+
+ spin_lock(&mali_mem_page_table_page_pool.lock);
+ if (0 < mali_mem_page_table_page_pool.count) {
+ u32 i = --mali_mem_page_table_page_pool.count;
+ *phys = mali_mem_page_table_page_pool.page[i].phys;
+ *mapping = mali_mem_page_table_page_pool.page[i].mapping;
+
+ ret = _MALI_OSK_ERR_OK;
+ }
+ spin_unlock(&mali_mem_page_table_page_pool.lock);
+
+ if (_MALI_OSK_ERR_OK != ret) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
+ *mapping = dma_alloc_attrs(&mali_platform_device->dev,
+ _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys,
+ GFP_KERNEL, &dma_attrs_wc);
+#else
+ *mapping = dma_alloc_writecombine(&mali_platform_device->dev,
+ _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys, GFP_KERNEL);
+#endif
+ if (NULL != *mapping) {
+ ret = _MALI_OSK_ERR_OK;
+
+#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT)
+ /* Verify that the "physical" address is 32-bit and
+ * usable for Mali, when on a system with bus addresses
+ * wider than 32-bit. */
+ MALI_DEBUG_ASSERT(0 == (tmp_phys >> 32));
+#endif
+
+ *phys = (mali_dma_addr)tmp_phys;
+ }
+ }
+
+ return ret;
+}
+
+void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt)
+{
+ spin_lock(&mali_mem_page_table_page_pool.lock);
+ if (MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE > mali_mem_page_table_page_pool.count) {
+ u32 i = mali_mem_page_table_page_pool.count;
+ mali_mem_page_table_page_pool.page[i].phys = phys;
+ mali_mem_page_table_page_pool.page[i].mapping = virt;
+
+ ++mali_mem_page_table_page_pool.count;
+
+ spin_unlock(&mali_mem_page_table_page_pool.lock);
+ } else {
+ spin_unlock(&mali_mem_page_table_page_pool.lock);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
+ dma_free_attrs(&mali_platform_device->dev,
+ _MALI_OSK_MALI_PAGE_SIZE, virt, phys,
+ &dma_attrs_wc);
+#else
+ dma_free_writecombine(&mali_platform_device->dev,
+ _MALI_OSK_MALI_PAGE_SIZE, virt, phys);
+#endif
+ }
+}
+
+void mali_mem_os_free_page_node(struct mali_page_node *m_page)
+{
+ struct page *page = m_page->page;
+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_OS);
+
+ if (1 == page_count(page)) {
+ dma_unmap_page(&mali_platform_device->dev, page_private(page),
+ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
+ ClearPagePrivate(page);
+ }
+ __free_page(page);
+ m_page->page = NULL;
+ list_del(&m_page->list);
+ kfree(m_page);
+}
+
+/* The maximum number of page table pool pages to free in one go. */
+#define MALI_MEM_OS_CHUNK_TO_FREE 64UL
+
+/* Free a certain number of pages from the page table page pool.
+ * The pool lock must be held when calling the function, and the lock will be
+ * released before returning.
+ */
+static void mali_mem_os_page_table_pool_free(size_t nr_to_free)
+{
+ mali_dma_addr phys_arr[MALI_MEM_OS_CHUNK_TO_FREE];
+ void *virt_arr[MALI_MEM_OS_CHUNK_TO_FREE];
+ u32 i;
+
+ MALI_DEBUG_ASSERT(nr_to_free <= MALI_MEM_OS_CHUNK_TO_FREE);
+
+ /* Remove nr_to_free pages from the pool and store them locally on stack. */
+ for (i = 0; i < nr_to_free; i++) {
+ u32 pool_index = mali_mem_page_table_page_pool.count - i - 1;
+
+ phys_arr[i] = mali_mem_page_table_page_pool.page[pool_index].phys;
+ virt_arr[i] = mali_mem_page_table_page_pool.page[pool_index].mapping;
+ }
+
+ mali_mem_page_table_page_pool.count -= nr_to_free;
+
+ spin_unlock(&mali_mem_page_table_page_pool.lock);
+
+ /* After releasing the spinlock: free the pages we removed from the pool. */
+ for (i = 0; i < nr_to_free; i++) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
+ dma_free_attrs(&mali_platform_device->dev, _MALI_OSK_MALI_PAGE_SIZE,
+ virt_arr[i], (dma_addr_t)phys_arr[i], &dma_attrs_wc);
+#else
+ dma_free_writecombine(&mali_platform_device->dev,
+ _MALI_OSK_MALI_PAGE_SIZE,
+ virt_arr[i], (dma_addr_t)phys_arr[i]);
+#endif
+ }
+}
+
+static void mali_mem_os_trim_page_table_page_pool(void)
+{
+ size_t nr_to_free = 0;
+ size_t nr_to_keep;
+
+ /* Keep 2 page table pages for each 1024 pages in the page cache. */
+ nr_to_keep = mali_mem_os_allocator.pool_count / 512;
+ /* And a minimum of eight pages, to accomodate new sessions. */
+ nr_to_keep += 8;
+
+ if (0 == spin_trylock(&mali_mem_page_table_page_pool.lock)) return;
+
+ if (nr_to_keep < mali_mem_page_table_page_pool.count) {
+ nr_to_free = mali_mem_page_table_page_pool.count - nr_to_keep;
+ nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, nr_to_free);
+ }
+
+ /* Pool lock will be released by the callee. */
+ mali_mem_os_page_table_pool_free(nr_to_free);
+}
+
+static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc)
+{
+ return mali_mem_os_allocator.pool_count;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
+static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask)
+#else
+static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask)
+#endif /* Linux < 2.6.35 */
+#else
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
+static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc)
+#else
+static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc)
+#endif /* Linux < 3.12.0 */
+#endif /* Linux < 3.0.0 */
+{
+ struct mali_page_node *m_page, *m_tmp;
+ unsigned long flags;
+ struct list_head *le, pages;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0)
+ int nr = nr_to_scan;
+#else
+ int nr = sc->nr_to_scan;
+#endif
+
+ if (0 == nr) {
+ return mali_mem_os_shrink_count(shrinker, sc);
+ }
+
+ if (0 == spin_trylock_irqsave(&mali_mem_os_allocator.pool_lock, flags)) {
+ /* Not able to lock. */
+ return -1;
+ }
+
+ if (0 == mali_mem_os_allocator.pool_count) {
+ /* No pages availble */
+ spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags);
+ return 0;
+ }
+
+ /* Release from general page pool */
+ nr = min((size_t)nr, mali_mem_os_allocator.pool_count);
+ mali_mem_os_allocator.pool_count -= nr;
+ list_for_each(le, &mali_mem_os_allocator.pool_pages) {
+ --nr;
+ if (0 == nr) break;
+ }
+ list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le);
+ spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags);
+
+ list_for_each_entry_safe(m_page, m_tmp, &pages, list) {
+ mali_mem_os_free_page_node(m_page);
+ }
+
+ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) {
+ /* Pools are empty, stop timer */
+ MALI_DEBUG_PRINT(5, ("Stopping timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count));
+ cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker);
+ }
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
+ return mali_mem_os_shrink_count(shrinker, sc);
+#else
+ return nr;
+#endif
+}
+
+static void mali_mem_os_trim_pool(struct work_struct *data)
+{
+ struct mali_page_node *m_page, *m_tmp;
+ struct list_head *le;
+ LIST_HEAD(pages);
+ size_t nr_to_free;
+
+ MALI_IGNORE(data);
+
+ MALI_DEBUG_PRINT(3, ("OS Mem: Trimming pool %u\n", mali_mem_os_allocator.pool_count));
+
+ /* Release from general page pool */
+ spin_lock(&mali_mem_os_allocator.pool_lock);
+ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) {
+ size_t count = mali_mem_os_allocator.pool_count - MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES;
+ const size_t min_to_free = min(64, MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES);
+
+ /* Free half the pages on the pool above the static limit. Or 64 pages, 256KB. */
+ nr_to_free = max(count / 2, min_to_free);
+
+ mali_mem_os_allocator.pool_count -= nr_to_free;
+ list_for_each(le, &mali_mem_os_allocator.pool_pages) {
+ --nr_to_free;
+ if (0 == nr_to_free) break;
+ }
+ list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le);
+ }
+ spin_unlock(&mali_mem_os_allocator.pool_lock);
+
+ list_for_each_entry_safe(m_page, m_tmp, &pages, list) {
+ mali_mem_os_free_page_node(m_page);
+ }
+
+ /* Release some pages from page table page pool */
+ mali_mem_os_trim_page_table_page_pool();
+
+ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) {
+ MALI_DEBUG_PRINT(4, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count));
+ queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES);
+ }
+}
+
+_mali_osk_errcode_t mali_mem_os_init(void)
+{
+ mali_mem_os_allocator.wq = alloc_workqueue("mali-mem", WQ_UNBOUND, 1);
+ if (NULL == mali_mem_os_allocator.wq) {
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
+ dma_set_attr(DMA_ATTR_WRITE_COMBINE, &dma_attrs_wc);
+#endif
+
+ register_shrinker(&mali_mem_os_allocator.shrinker);
+
+ return _MALI_OSK_ERR_OK;
+}
+
+void mali_mem_os_term(void)
+{
+ struct mali_page_node *m_page, *m_tmp;
+ unregister_shrinker(&mali_mem_os_allocator.shrinker);
+ cancel_delayed_work_sync(&mali_mem_os_allocator.timed_shrinker);
+
+ if (NULL != mali_mem_os_allocator.wq) {
+ destroy_workqueue(mali_mem_os_allocator.wq);
+ mali_mem_os_allocator.wq = NULL;
+ }
+
+ spin_lock(&mali_mem_os_allocator.pool_lock);
+ list_for_each_entry_safe(m_page, m_tmp, &mali_mem_os_allocator.pool_pages, list) {
+ mali_mem_os_free_page_node(m_page);
+
+ --mali_mem_os_allocator.pool_count;
+ }
+ BUG_ON(mali_mem_os_allocator.pool_count);
+ spin_unlock(&mali_mem_os_allocator.pool_lock);
+
+ /* Release from page table page pool */
+ do {
+ u32 nr_to_free;
+
+ spin_lock(&mali_mem_page_table_page_pool.lock);
+
+ nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, mali_mem_page_table_page_pool.count);
+
+ /* Pool lock will be released by the callee. */
+ mali_mem_os_page_table_pool_free(nr_to_free);
+ } while (0 != mali_mem_page_table_page_pool.count);
+}
+
+_mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size)
+{
+ mali_mem_os_allocator.allocation_limit = size;
+
+ MALI_SUCCESS;
+}
+
+u32 mali_mem_os_stat(void)
+{
+ return atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.h b/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.h
new file mode 100644
index 000000000000..f9ead166c455
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_MEMORY_OS_ALLOC_H__
+#define __MALI_MEMORY_OS_ALLOC_H__
+
+#include "mali_osk.h"
+#include "mali_memory_types.h"
+
+
+/** @brief Release Mali OS memory
+ *
+ * The session memory_lock must be held when calling this function.
+ *
+ * @param mem_bkend Pointer to the mali_mem_backend to release
+ */
+u32 mali_mem_os_release(mali_mem_backend *mem_bkend);
+
+_mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping);
+
+void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt);
+
+_mali_osk_errcode_t mali_mem_os_init(void);
+
+void mali_mem_os_term(void);
+
+u32 mali_mem_os_stat(void);
+
+void mali_mem_os_free_page_node(struct mali_page_node *m_page);
+
+int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size);
+
+u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag);
+
+_mali_osk_errcode_t mali_mem_os_put_page(struct page *page);
+
+_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count);
+
+_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props);
+
+void mali_mem_os_mali_unmap(mali_mem_allocation *alloc);
+
+int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma);
+
+_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size);
+
+#endif /* __MALI_MEMORY_OS_ALLOC_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.c b/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.c
new file mode 100644
index 000000000000..a46eb198c98f
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.c
@@ -0,0 +1,942 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/fs.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/version.h>
+#include <linux/sched.h>
+#include <linux/idr.h>
+#include <linux/platform_device.h>
+#include <linux/workqueue.h>
+#include <linux/shmem_fs.h>
+#include <linux/file.h>
+#include <linux/swap.h>
+#include <linux/pagemap.h>
+#include "mali_osk.h"
+#include "mali_osk_mali.h"
+#include "mali_memory.h"
+#include "mali_memory_manager.h"
+#include "mali_memory_virtual.h"
+#include "mali_memory_cow.h"
+#include "mali_ukk.h"
+#include "mali_kernel_utilization.h"
+#include "mali_memory_swap_alloc.h"
+
+
+static struct _mali_osk_bitmap idx_mgr;
+static struct file *global_swap_file;
+static struct address_space *global_swap_space;
+static _mali_osk_wq_work_t *mali_mem_swap_out_workq = NULL;
+static u32 mem_backend_swapped_pool_size;
+#ifdef MALI_MEM_SWAP_TRACKING
+static u32 mem_backend_swapped_unlock_size;
+#endif
+/* Lock order: mem_backend_swapped_pool_lock > each memory backend's mutex lock.
+ * This lock used to protect mem_backend_swapped_pool_size and mem_backend_swapped_pool. */
+static struct mutex mem_backend_swapped_pool_lock;
+static struct list_head mem_backend_swapped_pool;
+
+extern struct mali_mem_os_allocator mali_mem_os_allocator;
+
+#define MALI_SWAP_LOW_MEM_DEFAULT_VALUE (60*1024*1024)
+#define MALI_SWAP_INVALIDATE_MALI_ADDRESS (0) /* Used to mark the given memory cookie is invalidate. */
+#define MALI_SWAP_GLOBAL_SWAP_FILE_SIZE (0xFFFFFFFF)
+#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX ((MALI_SWAP_GLOBAL_SWAP_FILE_SIZE) >> PAGE_CACHE_SHIFT)
+#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE (1 << 15) /* Reserved for CoW nonlinear swap backend memory, the space size is 128MB. */
+
+unsigned int mali_mem_swap_out_threshold_value = MALI_SWAP_LOW_MEM_DEFAULT_VALUE;
+
+/**
+ * We have two situations to do shrinking things, one is we met low GPU utilization which shows GPU needn't touch too
+ * swappable backends in short time, and the other one is we add new swappable backends, the total pool size exceed
+ * the threshold value of the swapped pool size.
+ */
+typedef enum {
+ MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION = 100,
+ MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS = 257,
+} _mali_mem_swap_pool_shrink_type_t;
+
+static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg);
+
+_mali_osk_errcode_t mali_mem_swap_init(void)
+{
+ gfp_t flags = __GFP_NORETRY | __GFP_NOWARN;
+
+ if (_MALI_OSK_ERR_OK != _mali_osk_bitmap_init(&idx_mgr, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE)) {
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ global_swap_file = shmem_file_setup("mali_swap", MALI_SWAP_GLOBAL_SWAP_FILE_SIZE, VM_NORESERVE);
+ if (IS_ERR(global_swap_file)) {
+ _mali_osk_bitmap_term(&idx_mgr);
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ global_swap_space = global_swap_file->f_path.dentry->d_inode->i_mapping;
+
+ mali_mem_swap_out_workq = _mali_osk_wq_create_work(mali_mem_swap_swapped_bkend_pool_check_for_low_utilization, NULL);
+ if (NULL == mali_mem_swap_out_workq) {
+ _mali_osk_bitmap_term(&idx_mgr);
+ fput(global_swap_file);
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE)
+ flags |= GFP_HIGHUSER;
+#else
+#ifdef CONFIG_ZONE_DMA32
+ flags |= GFP_DMA32;
+#else
+#ifdef CONFIG_ZONE_DMA
+ flags |= GFP_DMA;
+#else
+ /* arm64 utgard only work on < 4G, but the kernel
+ * didn't provide method to allocte memory < 4G
+ */
+ MALI_DEBUG_ASSERT(0);
+#endif
+#endif
+#endif
+
+ /* When we use shmem_read_mapping_page to allocate/swap-in, it will
+ * use these flags to allocate new page if need.*/
+ mapping_set_gfp_mask(global_swap_space, flags);
+
+ mem_backend_swapped_pool_size = 0;
+#ifdef MALI_MEM_SWAP_TRACKING
+ mem_backend_swapped_unlock_size = 0;
+#endif
+ mutex_init(&mem_backend_swapped_pool_lock);
+ INIT_LIST_HEAD(&mem_backend_swapped_pool);
+
+ MALI_DEBUG_PRINT(2, ("Mali SWAP: Swap out threshold vaule is %uM\n", mali_mem_swap_out_threshold_value >> 20));
+
+ return _MALI_OSK_ERR_OK;
+}
+
+void mali_mem_swap_term(void)
+{
+ _mali_osk_bitmap_term(&idx_mgr);
+
+ fput(global_swap_file);
+
+ _mali_osk_wq_delete_work(mali_mem_swap_out_workq);
+
+ MALI_DEBUG_ASSERT(list_empty(&mem_backend_swapped_pool));
+ MALI_DEBUG_ASSERT(0 == mem_backend_swapped_pool_size);
+
+ return;
+}
+
+struct file *mali_mem_swap_get_global_swap_file(void)
+{
+ return global_swap_file;
+}
+
+/* Judge if swappable backend in swapped pool. */
+static mali_bool mali_memory_swap_backend_in_swapped_pool(mali_mem_backend *mem_bkend)
+{
+ MALI_DEBUG_ASSERT_POINTER(mem_bkend);
+
+ return !list_empty(&mem_bkend->list);
+}
+
+void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend)
+{
+ MALI_DEBUG_ASSERT_POINTER(mem_bkend);
+
+ mutex_lock(&mem_backend_swapped_pool_lock);
+ mutex_lock(&mem_bkend->mutex);
+
+ if (MALI_FALSE == mali_memory_swap_backend_in_swapped_pool(mem_bkend)) {
+ mutex_unlock(&mem_bkend->mutex);
+ mutex_unlock(&mem_backend_swapped_pool_lock);
+ return;
+ }
+
+ MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list));
+
+ list_del_init(&mem_bkend->list);
+
+ mutex_unlock(&mem_bkend->mutex);
+
+ mem_backend_swapped_pool_size -= mem_bkend->size;
+
+ mutex_unlock(&mem_backend_swapped_pool_lock);
+}
+
+static void mali_mem_swap_out_page_node(mali_page_node *page_node)
+{
+ MALI_DEBUG_ASSERT(page_node);
+
+ dma_unmap_page(&mali_platform_device->dev, page_node->swap_it->dma_addr,
+ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
+ set_page_dirty(page_node->swap_it->page);
+ page_cache_release(page_node->swap_it->page);
+}
+
+void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend)
+{
+ mali_page_node *m_page;
+
+ MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex));
+
+ if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN)) {
+ return;
+ }
+
+ mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN;
+
+ list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) {
+ mali_mem_swap_out_page_node(m_page);
+ }
+
+ return;
+}
+
+static void mali_mem_swap_unlock_partial_locked_mem_backend(mali_mem_backend *mem_bkend, mali_page_node *page_node)
+{
+ mali_page_node *m_page;
+
+ MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex));
+
+ list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) {
+ if (m_page == page_node) {
+ break;
+ }
+ mali_mem_swap_out_page_node(m_page);
+ }
+}
+
+static void mali_mem_swap_swapped_bkend_pool_shrink(_mali_mem_swap_pool_shrink_type_t shrink_type)
+{
+ mali_mem_backend *bkend, *tmp_bkend;
+ long system_free_size;
+ u32 last_gpu_utilization, gpu_utilization_threshold_value, temp_swap_out_threshold_value;
+
+ MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_backend_swapped_pool_lock));
+
+ if (MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION == shrink_type) {
+ /**
+ * When we met that system memory is very low and Mali locked swappable memory size is less than
+ * threshold value, and at the same time, GPU load is very low and don't need high performance,
+ * at this condition, we can unlock more swap memory backend from swapped backends pool.
+ */
+ gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION;
+ temp_swap_out_threshold_value = (mali_mem_swap_out_threshold_value >> 2);
+ } else {
+ /* When we add swappable memory backends to swapped pool, we need to think that we couldn't
+ * hold too much swappable backends in Mali driver, and also we need considering performance.
+ * So there is a balance for swapping out memory backend, we should follow the following conditions:
+ * 1. Total memory size in global mem backend swapped pool is more than the defined threshold value.
+ * 2. System level free memory size is less than the defined threshold value.
+ * 3. Please note that GPU utilization problem isn't considered in this condition.
+ */
+ gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS;
+ temp_swap_out_threshold_value = mali_mem_swap_out_threshold_value;
+ }
+
+ /* Get system free pages number. */
+ system_free_size = global_page_state(NR_FREE_PAGES) * PAGE_SIZE;
+ last_gpu_utilization = _mali_ukk_utilization_gp_pp();
+
+ if ((last_gpu_utilization < gpu_utilization_threshold_value)
+ && (system_free_size < mali_mem_swap_out_threshold_value)
+ && (mem_backend_swapped_pool_size > temp_swap_out_threshold_value)) {
+ list_for_each_entry_safe(bkend, tmp_bkend, &mem_backend_swapped_pool, list) {
+ if (mem_backend_swapped_pool_size <= temp_swap_out_threshold_value) {
+ break;
+ }
+
+ mutex_lock(&bkend->mutex);
+
+ /* check if backend is in use. */
+ if (0 < bkend->using_count) {
+ mutex_unlock(&bkend->mutex);
+ continue;
+ }
+
+ mali_mem_swap_unlock_single_mem_backend(bkend);
+ list_del_init(&bkend->list);
+ mem_backend_swapped_pool_size -= bkend->size;
+#ifdef MALI_MEM_SWAP_TRACKING
+ mem_backend_swapped_unlock_size += bkend->size;
+#endif
+ mutex_unlock(&bkend->mutex);
+ }
+ }
+
+ return;
+}
+
+static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg)
+{
+ MALI_IGNORE(arg);
+
+ mutex_lock(&mem_backend_swapped_pool_lock);
+
+ mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION);
+
+ mutex_unlock(&mem_backend_swapped_pool_lock);
+}
+
+/**
+ * After PP job finished, we add all of swappable memory backend used by this PP
+ * job to the tail of the global swapped pool, and if the total size of swappable memory is more than threshold
+ * value, we also need to shrink the swapped pool start from the head of the list.
+ */
+void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend)
+{
+ mutex_lock(&mem_backend_swapped_pool_lock);
+ mutex_lock(&mem_bkend->mutex);
+
+ if (mali_memory_swap_backend_in_swapped_pool(mem_bkend)) {
+ MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list));
+
+ list_del_init(&mem_bkend->list);
+ list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool);
+ mutex_unlock(&mem_bkend->mutex);
+ mutex_unlock(&mem_backend_swapped_pool_lock);
+ return;
+ }
+
+ list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool);
+
+ mutex_unlock(&mem_bkend->mutex);
+ mem_backend_swapped_pool_size += mem_bkend->size;
+
+ mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS);
+
+ mutex_unlock(&mem_backend_swapped_pool_lock);
+ return;
+}
+
+
+u32 mali_mem_swap_idx_alloc(void)
+{
+ return _mali_osk_bitmap_alloc(&idx_mgr);
+}
+
+void mali_mem_swap_idx_free(u32 idx)
+{
+ _mali_osk_bitmap_free(&idx_mgr, idx);
+}
+
+static u32 mali_mem_swap_idx_range_alloc(u32 count)
+{
+ u32 index;
+
+ index = _mali_osk_bitmap_alloc_range(&idx_mgr, count);
+
+ return index;
+}
+
+static void mali_mem_swap_idx_range_free(u32 idx, int num)
+{
+ _mali_osk_bitmap_free_range(&idx_mgr, idx, num);
+}
+
+struct mali_swap_item *mali_mem_swap_alloc_swap_item(void)
+{
+ mali_swap_item *swap_item;
+
+ swap_item = kzalloc(sizeof(mali_swap_item), GFP_KERNEL);
+
+ if (NULL == swap_item) {
+ return NULL;
+ }
+
+ atomic_set(&swap_item->ref_count, 1);
+ swap_item->page = NULL;
+ atomic_add(1, &mali_mem_os_allocator.allocated_pages);
+
+ return swap_item;
+}
+
+void mali_mem_swap_free_swap_item(mali_swap_item *swap_item)
+{
+ struct inode *file_node;
+ long long start, end;
+
+ /* If this swap item is shared, we just reduce the reference counter. */
+ if (0 == atomic_dec_return(&swap_item->ref_count)) {
+ file_node = global_swap_file->f_path.dentry->d_inode;
+ start = swap_item->idx;
+ start = start << 12;
+ end = start + PAGE_SIZE;
+
+ shmem_truncate_range(file_node, start, (end - 1));
+
+ mali_mem_swap_idx_free(swap_item->idx);
+
+ atomic_sub(1, &mali_mem_os_allocator.allocated_pages);
+
+ kfree(swap_item);
+ }
+}
+
+/* Used to allocate new swap item for new memory allocation and cow page for write. */
+struct mali_page_node *_mali_mem_swap_page_node_allocate(void)
+{
+ struct mali_page_node *m_page;
+
+ m_page = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP);
+
+ if (NULL == m_page) {
+ return NULL;
+ }
+
+ m_page->swap_it = mali_mem_swap_alloc_swap_item();
+
+ if (NULL == m_page->swap_it) {
+ kfree(m_page);
+ return NULL;
+ }
+
+ return m_page;
+}
+
+_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page)
+{
+
+ mali_mem_swap_free_swap_item(m_page->swap_it);
+
+ return _MALI_OSK_ERR_OK;
+}
+
+void _mali_mem_swap_page_node_free(struct mali_page_node *m_page)
+{
+ _mali_mem_swap_put_page_node(m_page);
+
+ kfree(m_page);
+
+ return;
+}
+
+u32 mali_mem_swap_free(mali_mem_swap *swap_mem)
+{
+ struct mali_page_node *m_page, *m_tmp;
+ u32 free_pages_nr = 0;
+
+ MALI_DEBUG_ASSERT_POINTER(swap_mem);
+
+ list_for_each_entry_safe(m_page, m_tmp, &swap_mem->pages, list) {
+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP);
+
+ /* free the page node and release the swap item, if the ref count is 1,
+ * then need also free the swap item. */
+ list_del(&m_page->list);
+ if (1 == _mali_page_node_get_ref_count(m_page)) {
+ free_pages_nr++;
+ }
+
+ _mali_mem_swap_page_node_free(m_page);
+ }
+
+ return free_pages_nr;
+}
+
+static u32 mali_mem_swap_cow_free(mali_mem_cow *cow_mem)
+{
+ struct mali_page_node *m_page, *m_tmp;
+ u32 free_pages_nr = 0;
+
+ MALI_DEBUG_ASSERT_POINTER(cow_mem);
+
+ list_for_each_entry_safe(m_page, m_tmp, &cow_mem->pages, list) {
+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP);
+
+ /* free the page node and release the swap item, if the ref count is 1,
+ * then need also free the swap item. */
+ list_del(&m_page->list);
+ if (1 == _mali_page_node_get_ref_count(m_page)) {
+ free_pages_nr++;
+ }
+
+ _mali_mem_swap_page_node_free(m_page);
+ }
+
+ return free_pages_nr;
+}
+
+u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped)
+{
+ mali_mem_allocation *alloc;
+ u32 free_pages_nr = 0;
+
+ MALI_DEBUG_ASSERT_POINTER(mem_bkend);
+ alloc = mem_bkend->mali_allocation;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+
+ if (is_mali_mapped) {
+ mali_mem_swap_mali_unmap(alloc);
+ }
+
+ mali_memory_swap_list_backend_delete(mem_bkend);
+
+ mutex_lock(&mem_bkend->mutex);
+ /* To make sure the given memory backend was unlocked from Mali side,
+ * and then free this memory block. */
+ mali_mem_swap_unlock_single_mem_backend(mem_bkend);
+ mutex_unlock(&mem_bkend->mutex);
+
+ if (MALI_MEM_SWAP == mem_bkend->type) {
+ free_pages_nr = mali_mem_swap_free(&mem_bkend->swap_mem);
+ } else {
+ free_pages_nr = mali_mem_swap_cow_free(&mem_bkend->cow_mem);
+ }
+
+ return free_pages_nr;
+}
+
+mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node)
+{
+ MALI_DEBUG_ASSERT(NULL != page_node);
+
+ page_node->swap_it->page = shmem_read_mapping_page(global_swap_space, page_node->swap_it->idx);
+
+ if (IS_ERR(page_node->swap_it->page)) {
+ MALI_DEBUG_PRINT_ERROR(("SWAP Mem: failed to swap in page with index: %d.\n", page_node->swap_it->idx));
+ return MALI_FALSE;
+ }
+
+ /* Ensure page is flushed from CPU caches. */
+ page_node->swap_it->dma_addr = dma_map_page(&mali_platform_device->dev, page_node->swap_it->page,
+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
+
+ return MALI_TRUE;
+}
+
+int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx)
+{
+ size_t page_count = PAGE_ALIGN(size) / PAGE_SIZE;
+ struct mali_page_node *m_page;
+ long system_free_size;
+ u32 i, index;
+ mali_bool ret;
+
+ MALI_DEBUG_ASSERT(NULL != swap_mem);
+ MALI_DEBUG_ASSERT(NULL != bkend_idx);
+ MALI_DEBUG_ASSERT(page_count <= MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE);
+
+ if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) {
+ MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n",
+ size,
+ atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE,
+ mali_mem_os_allocator.allocation_limit));
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ INIT_LIST_HEAD(&swap_mem->pages);
+ swap_mem->count = page_count;
+ index = mali_mem_swap_idx_range_alloc(page_count);
+
+ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == index) {
+ MALI_PRINT_ERROR(("Mali Swap: Failed to allocate continuous index for swappable Mali memory."));
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ for (i = 0; i < page_count; i++) {
+ m_page = _mali_mem_swap_page_node_allocate();
+
+ if (NULL == m_page) {
+ MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Failed to allocate mali page node."));
+ swap_mem->count = i;
+
+ mali_mem_swap_free(swap_mem);
+ mali_mem_swap_idx_range_free(index + i, page_count - i);
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ m_page->swap_it->idx = index + i;
+
+ ret = mali_mem_swap_in_page_node(m_page);
+
+ if (MALI_FALSE == ret) {
+ MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Allocate new page from SHMEM file failed."));
+ _mali_mem_swap_page_node_free(m_page);
+ mali_mem_swap_idx_range_free(index + i + 1, page_count - i - 1);
+
+ swap_mem->count = i;
+ mali_mem_swap_free(swap_mem);
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ list_add_tail(&m_page->list, &swap_mem->pages);
+ }
+
+ system_free_size = global_page_state(NR_FREE_PAGES) * PAGE_SIZE;
+
+ if ((system_free_size < mali_mem_swap_out_threshold_value)
+ && (mem_backend_swapped_pool_size > (mali_mem_swap_out_threshold_value >> 2))
+ && mali_utilization_enabled()) {
+ _mali_osk_wq_schedule_work(mali_mem_swap_out_workq);
+ }
+
+ *bkend_idx = index;
+ return 0;
+}
+
+void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc)
+{
+ struct mali_session_data *session;
+
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+ session = alloc->session;
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ mali_session_memory_lock(session);
+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start,
+ alloc->flags);
+ mali_session_memory_unlock(session);
+}
+
+
+/* Insert these pages from shmem to mali page table*/
+_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props)
+{
+ struct mali_page_directory *pagedir = session->page_directory;
+ struct mali_page_node *m_page;
+ dma_addr_t phys;
+ u32 virt = vaddr;
+ u32 prop = props;
+
+ list_for_each_entry(m_page, &swap_mem->pages, list) {
+ MALI_DEBUG_ASSERT(NULL != m_page->swap_it->page);
+ phys = m_page->swap_it->dma_addr;
+
+ mali_mmu_pagedir_update(pagedir, virt, phys, MALI_MMU_PAGE_SIZE, prop);
+ virt += MALI_MMU_PAGE_SIZE;
+ }
+
+ return _MALI_OSK_ERR_OK;
+}
+
+int mali_mem_swap_in_pages(struct mali_pp_job *job)
+{
+ u32 num_memory_cookies;
+ struct mali_session_data *session;
+ struct mali_vma_node *mali_vma_node = NULL;
+ mali_mem_allocation *mali_alloc = NULL;
+ mali_mem_backend *mem_bkend = NULL;
+ struct mali_page_node *m_page;
+ mali_bool swap_in_success = MALI_TRUE;
+ int i;
+
+ MALI_DEBUG_ASSERT_POINTER(job);
+
+ num_memory_cookies = mali_pp_job_num_memory_cookies(job);
+ session = mali_pp_job_get_session(job);
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ for (i = 0; i < num_memory_cookies; i++) {
+
+ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i);
+
+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
+ if (NULL == mali_vma_node) {
+ job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS;
+ swap_in_success = MALI_FALSE;
+ MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr));
+ continue;
+ }
+
+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
+ MALI_DEBUG_ASSERT(NULL != mali_alloc);
+
+ if (MALI_MEM_SWAP != mali_alloc->type &&
+ MALI_MEM_COW != mali_alloc->type) {
+ continue;
+ }
+
+ /* Get backend memory & Map on GPU */
+ mutex_lock(&mali_idr_mutex);
+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle);
+ mutex_unlock(&mali_idr_mutex);
+ MALI_DEBUG_ASSERT(NULL != mem_bkend);
+
+ /* We neednot hold backend's lock here, race safe.*/
+ if ((MALI_MEM_COW == mem_bkend->type) &&
+ (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) {
+ continue;
+ }
+
+ mutex_lock(&mem_bkend->mutex);
+
+ /* When swap_in_success is MALI_FALSE, it means this job has memory backend that could not be swapped in,
+ * and it will be aborted in mali scheduler, so here, we just mark those memory cookies which
+ * should not be swapped out when delete job to invalide */
+ if (MALI_FALSE == swap_in_success) {
+ job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS;
+ mutex_unlock(&mem_bkend->mutex);
+ continue;
+ }
+
+ /* Before swap in, checking if this memory backend has been swapped in by the latest flushed jobs. */
+ ++mem_bkend->using_count;
+
+ if (1 < mem_bkend->using_count) {
+ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags));
+ mutex_unlock(&mem_bkend->mutex);
+ continue;
+ }
+
+ if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)) {
+ mutex_unlock(&mem_bkend->mutex);
+ continue;
+ }
+
+
+ list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) {
+ if (MALI_FALSE == mali_mem_swap_in_page_node(m_page)) {
+ /* Don't have enough memory to swap in page, so release pages have already been swapped
+ * in and then mark this pp job to be fail. */
+ mali_mem_swap_unlock_partial_locked_mem_backend(mem_bkend, m_page);
+ swap_in_success = MALI_FALSE;
+ break;
+ }
+ }
+
+ if (swap_in_success) {
+#ifdef MALI_MEM_SWAP_TRACKING
+ mem_backend_swapped_unlock_size -= mem_bkend->size;
+#endif
+ _mali_osk_mutex_wait(session->memory_lock);
+ mali_mem_swap_mali_map(&mem_bkend->swap_mem, session, mali_alloc->mali_mapping.addr, mali_alloc->mali_mapping.properties);
+ _mali_osk_mutex_signal(session->memory_lock);
+
+ /* Remove the unlock flag from mem backend flags, mark this backend has been swapped in. */
+ mem_bkend->flags &= ~(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN);
+ mutex_unlock(&mem_bkend->mutex);
+ } else {
+ --mem_bkend->using_count;
+ /* Marking that this backend is not swapped in, need not to be processed anymore. */
+ job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS;
+ mutex_unlock(&mem_bkend->mutex);
+ }
+ }
+
+ job->swap_status = swap_in_success ? MALI_SWAP_IN_SUCC : MALI_SWAP_IN_FAIL;
+
+ return _MALI_OSK_ERR_OK;
+}
+
+int mali_mem_swap_out_pages(struct mali_pp_job *job)
+{
+ u32 num_memory_cookies;
+ struct mali_session_data *session;
+ struct mali_vma_node *mali_vma_node = NULL;
+ mali_mem_allocation *mali_alloc = NULL;
+ mali_mem_backend *mem_bkend = NULL;
+ int i;
+
+ MALI_DEBUG_ASSERT_POINTER(job);
+
+ num_memory_cookies = mali_pp_job_num_memory_cookies(job);
+ session = mali_pp_job_get_session(job);
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+
+ for (i = 0; i < num_memory_cookies; i++) {
+ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i);
+
+ if (MALI_SWAP_INVALIDATE_MALI_ADDRESS == mali_addr) {
+ continue;
+ }
+
+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0);
+
+ if (NULL == mali_vma_node) {
+ MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr));
+ continue;
+ }
+
+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
+ MALI_DEBUG_ASSERT(NULL != mali_alloc);
+
+ if (MALI_MEM_SWAP != mali_alloc->type &&
+ MALI_MEM_COW != mali_alloc->type) {
+ continue;
+ }
+
+ mutex_lock(&mali_idr_mutex);
+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle);
+ mutex_unlock(&mali_idr_mutex);
+ MALI_DEBUG_ASSERT(NULL != mem_bkend);
+
+ /* We neednot hold backend's lock here, race safe.*/
+ if ((MALI_MEM_COW == mem_bkend->type) &&
+ (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) {
+ continue;
+ }
+
+ mutex_lock(&mem_bkend->mutex);
+
+ MALI_DEBUG_ASSERT(0 < mem_bkend->using_count);
+
+ /* Reducing the using_count of mem backend means less pp job are using this memory backend,
+ * if this count get to zero, it means no pp job is using it now, could put it to swap out list. */
+ --mem_bkend->using_count;
+
+ if (0 < mem_bkend->using_count) {
+ mutex_unlock(&mem_bkend->mutex);
+ continue;
+ }
+ mutex_unlock(&mem_bkend->mutex);
+
+ mali_memory_swap_list_backend_add(mem_bkend);
+ }
+
+ return _MALI_OSK_ERR_OK;
+}
+
+int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep)
+{
+ struct mali_page_node *m_page, *found_node = NULL;
+ struct page *found_page;
+ mali_mem_swap *swap = NULL;
+ mali_mem_cow *cow = NULL;
+ dma_addr_t dma_addr;
+ u32 i = 0;
+
+ if (MALI_MEM_SWAP == mem_bkend->type) {
+ swap = &mem_bkend->swap_mem;
+ list_for_each_entry(m_page, &swap->pages, list) {
+ if (i == offset) {
+ found_node = m_page;
+ break;
+ }
+ i++;
+ }
+ } else {
+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type);
+ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags));
+
+ cow = &mem_bkend->cow_mem;
+ list_for_each_entry(m_page, &cow->pages, list) {
+ if (i == offset) {
+ found_node = m_page;
+ break;
+ }
+ i++;
+ }
+ }
+
+ if (NULL == found_node) {
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ found_page = shmem_read_mapping_page(global_swap_space, found_node->swap_it->idx);
+
+ if (!IS_ERR(found_page)) {
+ lock_page(found_page);
+ dma_addr = dma_map_page(&mali_platform_device->dev, found_page,
+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
+ dma_unmap_page(&mali_platform_device->dev, dma_addr,
+ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
+
+ *pagep = found_page;
+ } else {
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ return _MALI_OSK_ERR_OK;
+}
+
+int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep)
+{
+ struct mali_page_node *m_page, *found_node = NULL, *new_node = NULL;
+ mali_mem_cow *cow = NULL;
+ u32 i = 0;
+
+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type);
+ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED));
+ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags));
+ MALI_DEBUG_ASSERT(!mali_memory_swap_backend_in_swapped_pool(mem_bkend));
+
+ cow = &mem_bkend->cow_mem;
+ list_for_each_entry(m_page, &cow->pages, list) {
+ if (i == offset) {
+ found_node = m_page;
+ break;
+ }
+ i++;
+ }
+
+ if (NULL == found_node) {
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ new_node = _mali_mem_swap_page_node_allocate();
+
+ if (NULL == new_node) {
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ new_node->swap_it->idx = mali_mem_swap_idx_alloc();
+
+ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == new_node->swap_it->idx) {
+ MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW on demand.\n"));
+ kfree(new_node->swap_it);
+ kfree(new_node);
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ if (MALI_FALSE == mali_mem_swap_in_page_node(new_node)) {
+ _mali_mem_swap_page_node_free(new_node);
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ /* swap in found node for copy in kernel. */
+ if (MALI_FALSE == mali_mem_swap_in_page_node(found_node)) {
+ mali_mem_swap_out_page_node(new_node);
+ _mali_mem_swap_page_node_free(new_node);
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ _mali_mem_cow_copy_page(found_node, new_node);
+
+ list_replace(&found_node->list, &new_node->list);
+
+ if (1 != _mali_page_node_get_ref_count(found_node)) {
+ atomic_add(1, &mem_bkend->mali_allocation->session->mali_mem_allocated_pages);
+ if (atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > mem_bkend->mali_allocation->session->max_mali_mem_allocated_size) {
+ mem_bkend->mali_allocation->session->max_mali_mem_allocated_size = atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE;
+ }
+ mem_bkend->cow_mem.change_pages_nr++;
+ }
+
+ mali_mem_swap_out_page_node(found_node);
+ _mali_mem_swap_page_node_free(found_node);
+
+ /* When swap in the new page node, we have called dma_map_page for this page.\n */
+ dma_unmap_page(&mali_platform_device->dev, new_node->swap_it->dma_addr,
+ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE);
+
+ lock_page(new_node->swap_it->page);
+
+ *pagep = new_node->swap_it->page;
+
+ return _MALI_OSK_ERR_OK;
+}
+
+#ifdef MALI_MEM_SWAP_TRACKING
+void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size)
+{
+ *swap_pool_size = mem_backend_swapped_pool_size;
+ *unlock_size = mem_backend_swapped_unlock_size;
+}
+#endif
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.h b/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.h
new file mode 100644
index 000000000000..a393ecce3a00
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.h
@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_MEMORY_SWAP_ALLOC_H__
+#define __MALI_MEMORY_SWAP_ALLOC_H__
+
+#include "mali_osk.h"
+#include "mali_session.h"
+
+#include "mali_memory_types.h"
+#include "mali_pp_job.h"
+
+/**
+ * Initialize memory swapping module.
+ */
+_mali_osk_errcode_t mali_mem_swap_init(void);
+
+void mali_mem_swap_term(void);
+
+/**
+ * Return global share memory file to other modules.
+ */
+struct file *mali_mem_swap_get_global_swap_file(void);
+
+/**
+ * Unlock the given memory backend and pages in it could be swapped out by kernel.
+ */
+void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend);
+
+/**
+ * Remove the given memory backend from global swap list.
+ */
+void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend);
+
+/**
+ * Add the given memory backend to global swap list.
+ */
+void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend);
+
+/**
+ * Allocate 1 index from bitmap used as page index in global swap file.
+ */
+u32 mali_mem_swap_idx_alloc(void);
+
+void mali_mem_swap_idx_free(u32 idx);
+
+/**
+ * Allocate a new swap item without page index.
+ */
+struct mali_swap_item *mali_mem_swap_alloc_swap_item(void);
+
+/**
+ * Free a swap item, truncate the corresponding space in page cache and free index of page.
+ */
+void mali_mem_swap_free_swap_item(mali_swap_item *swap_item);
+
+/**
+ * Allocate a page node with swap item.
+ */
+struct mali_page_node *_mali_mem_swap_page_node_allocate(void);
+
+/**
+ * Reduce the reference count of given page node and if return 0, just free this page node.
+ */
+_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page);
+
+void _mali_mem_swap_page_node_free(struct mali_page_node *m_page);
+
+/**
+ * Free a swappable memory backend.
+ */
+u32 mali_mem_swap_free(mali_mem_swap *swap_mem);
+
+/**
+ * Ummap and free.
+ */
+u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped);
+
+/**
+ * Read in a page from global swap file with the pre-allcated page index.
+ */
+mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node);
+
+int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx);
+
+_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props);
+
+void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc);
+
+/**
+ * When pp job created, we need swap in all of memory backend needed by this pp job.
+ */
+int mali_mem_swap_in_pages(struct mali_pp_job *job);
+
+/**
+ * Put all of memory backends used this pp job to the global swap list.
+ */
+int mali_mem_swap_out_pages(struct mali_pp_job *job);
+
+/**
+ * This will be called in page fault to process CPU read&write.
+ */
+int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) ;
+
+/**
+ * Used to process cow on demand for swappable memory backend.
+ */
+int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep);
+
+#ifdef MALI_MEM_SWAP_TRACKING
+void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size);
+#endif
+#endif /* __MALI_MEMORY_SWAP_ALLOC_H__ */
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_types.h b/drivers/gpu/arm/utgard/linux/mali_memory_types.h
new file mode 100644
index 000000000000..82af8fed66c9
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_types.h
@@ -0,0 +1,208 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_MEMORY_TYPES_H__
+#define __MALI_MEMORY_TYPES_H__
+
+#include <linux/mm.h>
+
+#if defined(CONFIG_MALI400_UMP)
+#include "ump_kernel_interface.h"
+#endif
+
+typedef u32 mali_address_t;
+
+typedef enum mali_mem_type {
+ MALI_MEM_OS,
+ MALI_MEM_EXTERNAL,
+ MALI_MEM_SWAP,
+ MALI_MEM_DMA_BUF,
+ MALI_MEM_UMP,
+ MALI_MEM_BLOCK,
+ MALI_MEM_COW,
+ MALI_MEM_TYPE_MAX,
+} mali_mem_type;
+
+typedef struct mali_block_item {
+ /* for block type, the block_phy is alway page size align
+ * so use low 12bit used for ref_cout.
+ */
+ unsigned long phy_addr;
+} mali_block_item;
+
+/**
+ * idx is used to locate the given page in the address space of swap file.
+ * ref_count is used to mark how many memory backends are using this item.
+ */
+typedef struct mali_swap_item {
+ u32 idx;
+ atomic_t ref_count;
+ struct page *page;
+ dma_addr_t dma_addr;
+} mali_swap_item;
+
+typedef enum mali_page_node_type {
+ MALI_PAGE_NODE_OS,
+ MALI_PAGE_NODE_BLOCK,
+ MALI_PAGE_NODE_SWAP,
+} mali_page_node_type;
+
+typedef struct mali_page_node {
+ struct list_head list;
+ union {
+ struct page *page;
+ mali_block_item *blk_it; /*pointer to block item*/
+ mali_swap_item *swap_it;
+ };
+
+ u32 type;
+} mali_page_node;
+
+typedef struct mali_mem_os_mem {
+ struct list_head pages;
+ u32 count;
+} mali_mem_os_mem;
+
+typedef struct mali_mem_dma_buf {
+#if defined(CONFIG_DMA_SHARED_BUFFER)
+ struct mali_dma_buf_attachment *attachment;
+#endif
+} mali_mem_dma_buf;
+
+typedef struct mali_mem_external {
+ dma_addr_t phys;
+ u32 size;
+} mali_mem_external;
+
+typedef struct mali_mem_ump {
+#if defined(CONFIG_MALI400_UMP)
+ ump_dd_handle handle;
+#endif
+} mali_mem_ump;
+
+typedef struct block_allocator_allocation {
+ /* The list will be released in reverse order */
+ struct block_info *last_allocated;
+ u32 mapping_length;
+ struct block_allocator *info;
+} block_allocator_allocation;
+
+typedef struct mali_mem_block_mem {
+ struct list_head pfns;
+ u32 count;
+} mali_mem_block_mem;
+
+typedef struct mali_mem_virt_mali_mapping {
+ mali_address_t addr; /* Virtual Mali address */
+ u32 properties; /* MMU Permissions + cache, must match MMU HW */
+} mali_mem_virt_mali_mapping;
+
+typedef struct mali_mem_virt_cpu_mapping {
+ void __user *addr;
+ struct vm_area_struct *vma;
+} mali_mem_virt_cpu_mapping;
+
+#define MALI_MEM_ALLOCATION_VALID_MAGIC 0xdeda110c
+#define MALI_MEM_ALLOCATION_FREED_MAGIC 0x10101010
+
+typedef struct mali_mm_node {
+ /* MALI GPU vaddr start, use u32 for mmu only support 32bit address*/
+ uint32_t start; /* GPU vaddr */
+ uint32_t size; /* GPU allocation virtual size */
+ unsigned allocated : 1;
+} mali_mm_node;
+
+typedef struct mali_vma_node {
+ struct mali_mm_node vm_node;
+ struct rb_node vm_rb;
+} mali_vma_node;
+
+
+typedef struct mali_mem_allocation {
+ MALI_DEBUG_CODE(u32 magic);
+ mali_mem_type type; /**< Type of memory */
+ u32 flags; /**< Flags for this allocation */
+
+ struct mali_session_data *session; /**< Pointer to session that owns the allocation */
+
+ mali_mem_virt_cpu_mapping cpu_mapping; /**< CPU mapping */
+ mali_mem_virt_mali_mapping mali_mapping; /**< Mali mapping */
+
+ /* add for new memory system */
+ struct mali_vma_node mali_vma_node;
+ u32 vsize; /* virtual size*/
+ u32 psize; /* physical backend memory size*/
+ struct list_head list;
+ s32 backend_handle; /* idr for mem_backend */
+ _mali_osk_atomic_t mem_alloc_refcount;
+} mali_mem_allocation;
+
+struct mali_mem_os_allocator {
+ spinlock_t pool_lock;
+ struct list_head pool_pages;
+ size_t pool_count;
+
+ atomic_t allocated_pages;
+ size_t allocation_limit;
+
+ struct shrinker shrinker;
+ struct delayed_work timed_shrinker;
+ struct workqueue_struct *wq;
+};
+
+/* COW backend memory type */
+typedef struct mali_mem_cow {
+ struct list_head pages; /**< all pages for this cow backend allocation,
+ including new allocated pages for modified range*/
+ u32 count; /**< number of pages */
+ s32 change_pages_nr;
+} mali_mem_cow;
+
+typedef struct mali_mem_swap {
+ struct list_head pages;
+ u32 count;
+} mali_mem_swap;
+
+#define MALI_MEM_BACKEND_FLAG_COWED (0x1) /* COW has happen on this backend */
+#define MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE (0x2) /* This is an COW backend, mapped as not allowed cpu to write */
+#define MALI_MEM_BACKEND_FLAG_SWAP_COWED (0x4) /* Mark the given backend is cowed from swappable memory. */
+/* Mark this backend is not swapped_in in MALI driver, and before using it,
+ * we should swap it in and set up corresponding page table. */
+#define MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN (0x8)
+#define MALI_MEM_BACKEND_FLAG_NOT_BINDED (0x1 << 5) /* this backend it not back with physical memory, used for defer bind */
+#define MALI_MEM_BACKEND_FLAG_BINDED (0x1 << 6) /* this backend it back with physical memory, used for defer bind */
+
+typedef struct mali_mem_backend {
+ mali_mem_type type; /**< Type of backend memory */
+ u32 flags; /**< Flags for this allocation */
+ u32 size;
+ /* Union selected by type. */
+ union {
+ mali_mem_os_mem os_mem; /**< MALI_MEM_OS */
+ mali_mem_external ext_mem; /**< MALI_MEM_EXTERNAL */
+ mali_mem_dma_buf dma_buf; /**< MALI_MEM_DMA_BUF */
+ mali_mem_ump ump_mem; /**< MALI_MEM_UMP */
+ mali_mem_block_mem block_mem; /**< MALI_MEM_BLOCK */
+ mali_mem_cow cow_mem;
+ mali_mem_swap swap_mem;
+ };
+ mali_mem_allocation *mali_allocation;
+ struct mutex mutex;
+ mali_mem_type cow_type;
+
+ struct list_head list; /**< Used to link swappable memory backend to the global swappable list */
+ int using_count; /**< Mark how many PP jobs are using this memory backend */
+ u32 start_idx; /**< If the correspondign vma of this backend is linear, this value will be used to set vma->vm_pgoff */
+} mali_mem_backend;
+
+#define MALI_MEM_FLAG_MALI_GUARD_PAGE (_MALI_MAP_EXTERNAL_MAP_GUARD_PAGE)
+#define MALI_MEM_FLAG_DONT_CPU_MAP (1 << 1)
+#define MALI_MEM_FLAG_CAN_RESIZE (_MALI_MEMORY_ALLOCATE_RESIZEABLE)
+#endif /* __MALI_MEMORY_TYPES__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_ump.c b/drivers/gpu/arm/utgard/linux/mali_memory_ump.c
new file mode 100644
index 000000000000..bacadc85e47e
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_ump.c
@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include "mali_ukk.h"
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_kernel_linux.h"
+#include "mali_memory.h"
+#include "ump_kernel_interface.h"
+
+static int mali_mem_ump_map(mali_mem_backend *mem_backend)
+{
+ ump_dd_handle ump_mem;
+ mali_mem_allocation *alloc;
+ struct mali_session_data *session;
+ u32 nr_blocks;
+ u32 i;
+ ump_dd_physical_block *ump_blocks;
+ struct mali_page_directory *pagedir;
+ u32 offset = 0;
+ _mali_osk_errcode_t err;
+
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+ MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type);
+
+ alloc = mem_backend->mali_allocation;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+
+ session = alloc->session;
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ ump_mem = mem_backend->ump_mem.handle;
+ MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem);
+
+ nr_blocks = ump_dd_phys_block_count_get(ump_mem);
+ if (nr_blocks == 0) {
+ MALI_DEBUG_PRINT(1, ("No block count\n"));
+ return -EINVAL;
+ }
+
+ ump_blocks = _mali_osk_malloc(sizeof(*ump_blocks) * nr_blocks);
+ if (NULL == ump_blocks) {
+ return -ENOMEM;
+ }
+
+ if (UMP_DD_INVALID == ump_dd_phys_blocks_get(ump_mem, ump_blocks, nr_blocks)) {
+ _mali_osk_free(ump_blocks);
+ return -EFAULT;
+ }
+
+ pagedir = session->page_directory;
+
+ mali_session_memory_lock(session);
+
+ err = mali_mem_mali_map_prepare(alloc);
+ if (_MALI_OSK_ERR_OK != err) {
+ MALI_DEBUG_PRINT(1, ("Mapping of UMP memory failed\n"));
+
+ _mali_osk_free(ump_blocks);
+ mali_session_memory_unlock(session);
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < nr_blocks; ++i) {
+ u32 virt = alloc->mali_vma_node.vm_node.start + offset;
+
+ MALI_DEBUG_PRINT(7, ("Mapping in 0x%08x size %d\n", ump_blocks[i].addr , ump_blocks[i].size));
+
+ mali_mmu_pagedir_update(pagedir, virt, ump_blocks[i].addr,
+ ump_blocks[i].size, MALI_MMU_FLAGS_DEFAULT);
+
+ offset += ump_blocks[i].size;
+ }
+
+ if (alloc->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) {
+ u32 virt = alloc->mali_vma_node.vm_node.start + offset;
+
+ /* Map in an extra virtual guard page at the end of the VMA */
+ MALI_DEBUG_PRINT(6, ("Mapping in extra guard page\n"));
+
+ mali_mmu_pagedir_update(pagedir, virt, ump_blocks[0].addr, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT);
+
+ offset += _MALI_OSK_MALI_PAGE_SIZE;
+ }
+ mali_session_memory_unlock(session);
+ _mali_osk_free(ump_blocks);
+ return 0;
+}
+
+static void mali_mem_ump_unmap(mali_mem_allocation *alloc)
+{
+ struct mali_session_data *session;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+ session = alloc->session;
+ MALI_DEBUG_ASSERT_POINTER(session);
+ mali_session_memory_lock(session);
+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start,
+ alloc->flags);
+ mali_session_memory_unlock(session);
+}
+
+int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags)
+{
+ ump_dd_handle ump_mem;
+ int ret;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+ MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type);
+
+ MALI_DEBUG_PRINT(3,
+ ("Requested to map ump memory with secure id %d into virtual memory 0x%08X, size 0x%08X\n",
+ secure_id, alloc->mali_vma_node.vm_node.start, alloc->mali_vma_node.vm_node.size));
+
+ ump_mem = ump_dd_handle_create_from_secure_id(secure_id);
+ if (UMP_DD_HANDLE_INVALID == ump_mem) MALI_ERROR(_MALI_OSK_ERR_FAULT);
+ alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP;
+ if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) {
+ alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE;
+ }
+
+ mem_backend->ump_mem.handle = ump_mem;
+
+ ret = mali_mem_ump_map(mem_backend);
+ if (0 != ret) {
+ ump_dd_reference_release(ump_mem);
+ return _MALI_OSK_ERR_FAULT;
+ }
+ MALI_DEBUG_PRINT(3, ("Returning from UMP bind\n"));
+ return _MALI_OSK_ERR_OK;
+}
+
+void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend)
+{
+ ump_dd_handle ump_mem;
+ mali_mem_allocation *alloc;
+ MALI_DEBUG_ASSERT_POINTER(mem_backend);
+ MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type);
+ ump_mem = mem_backend->ump_mem.handle;
+ MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem);
+
+ alloc = mem_backend->mali_allocation;
+ MALI_DEBUG_ASSERT_POINTER(alloc);
+ mali_mem_ump_unmap(alloc);
+ ump_dd_reference_release(ump_mem);
+}
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_ump.h b/drivers/gpu/arm/utgard/linux/mali_memory_ump.h
new file mode 100644
index 000000000000..4adb70df884e
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_ump.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_MEMORY_UMP_BUF_H__
+#define __MALI_MEMORY_UMP_BUF_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "mali_uk_types.h"
+#include "mali_osk.h"
+#include "mali_memory.h"
+
+int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags);
+void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __MALI_MEMORY_DMA_BUF_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_util.c b/drivers/gpu/arm/utgard/linux/mali_memory_util.c
new file mode 100644
index 000000000000..bcca43c09f0b
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_util.c
@@ -0,0 +1,149 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/fs.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+
+#include "mali_osk.h"
+#include "mali_osk_mali.h"
+#include "mali_kernel_linux.h"
+#include "mali_scheduler.h"
+
+#include "mali_memory.h"
+#include "mali_memory_os_alloc.h"
+#if defined(CONFIG_DMA_SHARED_BUFFER)
+#include "mali_memory_dma_buf.h"
+#endif
+#if defined(CONFIG_MALI400_UMP)
+#include "mali_memory_ump.h"
+#endif
+#include "mali_memory_external.h"
+#include "mali_memory_manager.h"
+#include "mali_memory_virtual.h"
+#include "mali_memory_cow.h"
+#include "mali_memory_block_alloc.h"
+#include "mali_memory_swap_alloc.h"
+
+
+
+/**
+*function @_mali_free_allocation_mem - free a memory allocation
+*/
+static u32 _mali_free_allocation_mem(mali_mem_allocation *mali_alloc)
+{
+ mali_mem_backend *mem_bkend = NULL;
+ u32 free_pages_nr = 0;
+
+ struct mali_session_data *session = mali_alloc->session;
+ MALI_DEBUG_PRINT(4, (" _mali_free_allocation_mem, psize =0x%x! \n", mali_alloc->psize));
+ if (0 == mali_alloc->psize)
+ goto out;
+
+ /* Get backend memory & Map on CPU */
+ mutex_lock(&mali_idr_mutex);
+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle);
+ mutex_unlock(&mali_idr_mutex);
+ MALI_DEBUG_ASSERT(NULL != mem_bkend);
+
+ switch (mem_bkend->type) {
+ case MALI_MEM_OS:
+ free_pages_nr = mali_mem_os_release(mem_bkend);
+ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages);
+ break;
+ case MALI_MEM_UMP:
+#if defined(CONFIG_MALI400_UMP)
+ mali_mem_unbind_ump_buf(mem_bkend);
+ atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]);
+#else
+ MALI_DEBUG_PRINT(2, ("UMP not supported\n"));
+#endif
+ break;
+ case MALI_MEM_DMA_BUF:
+#if defined(CONFIG_DMA_SHARED_BUFFER)
+ mali_mem_unbind_dma_buf(mem_bkend);
+ atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]);
+#else
+ MALI_DEBUG_PRINT(2, ("DMA not supported\n"));
+#endif
+ break;
+ case MALI_MEM_EXTERNAL:
+ mali_mem_unbind_ext_buf(mem_bkend);
+ atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]);
+ break;
+
+ case MALI_MEM_BLOCK:
+ free_pages_nr = mali_mem_block_release(mem_bkend);
+ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages);
+ break;
+
+ case MALI_MEM_COW:
+ if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) {
+ free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE);
+ } else {
+ free_pages_nr = mali_mem_cow_release(mem_bkend, MALI_TRUE);
+ }
+ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages);
+ break;
+ case MALI_MEM_SWAP:
+ free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE);
+ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages);
+ atomic_sub(free_pages_nr, &session->mali_mem_array[mem_bkend->type]);
+ break;
+ default:
+ MALI_DEBUG_PRINT(1, ("mem type %d is not in the mali_mem_type enum.\n", mem_bkend->type));
+ break;
+ }
+
+ /*Remove backend memory idex */
+ mutex_lock(&mali_idr_mutex);
+ idr_remove(&mali_backend_idr, mali_alloc->backend_handle);
+ mutex_unlock(&mali_idr_mutex);
+ kfree(mem_bkend);
+out:
+ /* remove memory allocation */
+ mali_vma_offset_remove(&session->allocation_mgr, &mali_alloc->mali_vma_node);
+ mali_mem_allocation_struct_destory(mali_alloc);
+ return free_pages_nr;
+}
+
+/**
+* ref_count for allocation
+*/
+u32 mali_allocation_unref(struct mali_mem_allocation **alloc)
+{
+ u32 free_pages_nr = 0;
+ mali_mem_allocation *mali_alloc = *alloc;
+ *alloc = NULL;
+ if (0 == _mali_osk_atomic_dec_return(&mali_alloc->mem_alloc_refcount)) {
+ free_pages_nr = _mali_free_allocation_mem(mali_alloc);
+ }
+ return free_pages_nr;
+}
+
+void mali_allocation_ref(struct mali_mem_allocation *alloc)
+{
+ _mali_osk_atomic_inc(&alloc->mem_alloc_refcount);
+}
+
+void mali_free_session_allocations(struct mali_session_data *session)
+{
+ struct mali_mem_allocation *entry, *next;
+
+ MALI_DEBUG_PRINT(4, (" mali_free_session_allocations! \n"));
+
+ list_for_each_entry_safe(entry, next, &session->allocation_mgr.head, list) {
+ mali_allocation_unref(&entry);
+ }
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_util.h b/drivers/gpu/arm/utgard/linux/mali_memory_util.h
new file mode 100644
index 000000000000..0d686979d7a8
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_util.h
@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_MEMORY_UTIL_H__
+#define __MALI_MEMORY_UTIL_H__
+
+u32 mali_allocation_unref(struct mali_mem_allocation **alloc);
+
+void mali_allocation_ref(struct mali_mem_allocation *alloc);
+
+void mali_free_session_allocations(struct mali_session_data *session);
+
+#endif
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_virtual.c b/drivers/gpu/arm/utgard/linux/mali_memory_virtual.c
new file mode 100644
index 000000000000..35f8f90a444d
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_virtual.c
@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/fs.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+
+#include "mali_osk.h"
+#include "mali_osk_mali.h"
+#include "mali_kernel_linux.h"
+#include "mali_scheduler.h"
+#include "mali_memory_os_alloc.h"
+#include "mali_memory_manager.h"
+#include "mali_memory_virtual.h"
+
+
+/**
+*internal helper to link node into the rb-tree
+*/
+static inline void _mali_vma_offset_add_rb(struct mali_allocation_manager *mgr,
+ struct mali_vma_node *node)
+{
+ struct rb_node **iter = &mgr->allocation_mgr_rb.rb_node;
+ struct rb_node *parent = NULL;
+ struct mali_vma_node *iter_node;
+
+ while (likely(*iter)) {
+ parent = *iter;
+ iter_node = rb_entry(*iter, struct mali_vma_node, vm_rb);
+
+ if (node->vm_node.start < iter_node->vm_node.start)
+ iter = &(*iter)->rb_left;
+ else if (node->vm_node.start > iter_node->vm_node.start)
+ iter = &(*iter)->rb_right;
+ else
+ MALI_DEBUG_ASSERT(0);
+ }
+
+ rb_link_node(&node->vm_rb, parent, iter);
+ rb_insert_color(&node->vm_rb, &mgr->allocation_mgr_rb);
+}
+
+/**
+ * mali_vma_offset_add() - Add offset node to RB Tree
+ */
+int mali_vma_offset_add(struct mali_allocation_manager *mgr,
+ struct mali_vma_node *node)
+{
+ int ret = 0;
+ write_lock(&mgr->vm_lock);
+
+ if (node->vm_node.allocated) {
+ goto out;
+ }
+
+ _mali_vma_offset_add_rb(mgr, node);
+ /* set to allocated */
+ node->vm_node.allocated = 1;
+
+out:
+ write_unlock(&mgr->vm_lock);
+ return ret;
+}
+
+/**
+ * mali_vma_offset_remove() - Remove offset node from RB tree
+ */
+void mali_vma_offset_remove(struct mali_allocation_manager *mgr,
+ struct mali_vma_node *node)
+{
+ write_lock(&mgr->vm_lock);
+
+ if (node->vm_node.allocated) {
+ rb_erase(&node->vm_rb, &mgr->allocation_mgr_rb);
+ memset(&node->vm_node, 0, sizeof(node->vm_node));
+ }
+ write_unlock(&mgr->vm_lock);
+}
+
+/**
+* mali_vma_offset_search - Search the node in RB tree
+*/
+struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr,
+ unsigned long start, unsigned long pages)
+{
+ struct mali_vma_node *node, *best;
+ struct rb_node *iter;
+ unsigned long offset;
+ read_lock(&mgr->vm_lock);
+
+ iter = mgr->allocation_mgr_rb.rb_node;
+ best = NULL;
+
+ while (likely(iter)) {
+ node = rb_entry(iter, struct mali_vma_node, vm_rb);
+ offset = node->vm_node.start;
+ if (start >= offset) {
+ iter = iter->rb_right;
+ best = node;
+ if (start == offset)
+ break;
+ } else {
+ iter = iter->rb_left;
+ }
+ }
+
+ if (best) {
+ offset = best->vm_node.start + best->vm_node.size;
+ if (offset <= start + pages)
+ best = NULL;
+ }
+ read_unlock(&mgr->vm_lock);
+
+ return best;
+}
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_virtual.h b/drivers/gpu/arm/utgard/linux/mali_memory_virtual.h
new file mode 100644
index 000000000000..3e2d48f44e28
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_memory_virtual.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#ifndef __MALI_GPU_VMEM_H__
+#define __MALI_GPU_VMEM_H__
+
+#include "mali_osk.h"
+#include "mali_session.h"
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/rbtree.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+#include "mali_memory_types.h"
+#include "mali_memory_os_alloc.h"
+#include "mali_memory_manager.h"
+
+
+
+int mali_vma_offset_add(struct mali_allocation_manager *mgr,
+ struct mali_vma_node *node);
+
+void mali_vma_offset_remove(struct mali_allocation_manager *mgr,
+ struct mali_vma_node *node);
+
+struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr,
+ unsigned long start, unsigned long pages);
+
+#endif
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_atomics.c b/drivers/gpu/arm/utgard/linux/mali_osk_atomics.c
new file mode 100644
index 000000000000..ba630e2730cf
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_atomics.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_atomics.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+
+#include "mali_osk.h"
+#include <asm/atomic.h>
+#include "mali_kernel_common.h"
+
+void _mali_osk_atomic_dec(_mali_osk_atomic_t *atom)
+{
+ atomic_dec((atomic_t *)&atom->u.val);
+}
+
+u32 _mali_osk_atomic_dec_return(_mali_osk_atomic_t *atom)
+{
+ return atomic_dec_return((atomic_t *)&atom->u.val);
+}
+
+void _mali_osk_atomic_inc(_mali_osk_atomic_t *atom)
+{
+ atomic_inc((atomic_t *)&atom->u.val);
+}
+
+u32 _mali_osk_atomic_inc_return(_mali_osk_atomic_t *atom)
+{
+ return atomic_inc_return((atomic_t *)&atom->u.val);
+}
+
+void _mali_osk_atomic_init(_mali_osk_atomic_t *atom, u32 val)
+{
+ MALI_DEBUG_ASSERT_POINTER(atom);
+ atomic_set((atomic_t *)&atom->u.val, val);
+}
+
+u32 _mali_osk_atomic_read(_mali_osk_atomic_t *atom)
+{
+ return atomic_read((atomic_t *)&atom->u.val);
+}
+
+void _mali_osk_atomic_term(_mali_osk_atomic_t *atom)
+{
+ MALI_IGNORE(atom);
+}
+
+u32 _mali_osk_atomic_xchg(_mali_osk_atomic_t *atom, u32 val)
+{
+ return atomic_xchg((atomic_t *)&atom->u.val, val);
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_bitmap.c b/drivers/gpu/arm/utgard/linux/mali_osk_bitmap.c
new file mode 100644
index 000000000000..01ca38235b20
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_bitmap.c
@@ -0,0 +1,152 @@
+/*
+ * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_bitmap.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+#include <linux/bitmap.h>
+#include <linux/vmalloc.h>
+#include "common/mali_kernel_common.h"
+#include "mali_osk_types.h"
+#include "mali_osk.h"
+
+u32 _mali_osk_bitmap_alloc(struct _mali_osk_bitmap *bitmap)
+{
+ u32 obj;
+
+ MALI_DEBUG_ASSERT_POINTER(bitmap);
+
+ _mali_osk_spinlock_lock(bitmap->lock);
+
+ obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->reserve);
+
+ if (obj < bitmap->max) {
+ set_bit(obj, bitmap->table);
+ } else {
+ obj = -1;
+ }
+
+ if (obj != -1)
+ --bitmap->avail;
+ _mali_osk_spinlock_unlock(bitmap->lock);
+
+ return obj;
+}
+
+void _mali_osk_bitmap_free(struct _mali_osk_bitmap *bitmap, u32 obj)
+{
+ MALI_DEBUG_ASSERT_POINTER(bitmap);
+
+ _mali_osk_bitmap_free_range(bitmap, obj, 1);
+}
+
+u32 _mali_osk_bitmap_alloc_range(struct _mali_osk_bitmap *bitmap, int cnt)
+{
+ u32 obj;
+
+ MALI_DEBUG_ASSERT_POINTER(bitmap);
+
+ if (0 >= cnt) {
+ return -1;
+ }
+
+ if (1 == cnt) {
+ return _mali_osk_bitmap_alloc(bitmap);
+ }
+
+ _mali_osk_spinlock_lock(bitmap->lock);
+ obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
+ bitmap->last, cnt, 0);
+
+ if (obj >= bitmap->max) {
+ obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
+ bitmap->reserve, cnt, 0);
+ }
+
+ if (obj < bitmap->max) {
+ bitmap_set(bitmap->table, obj, cnt);
+
+ bitmap->last = (obj + cnt);
+ if (bitmap->last >= bitmap->max) {
+ bitmap->last = bitmap->reserve;
+ }
+ } else {
+ obj = -1;
+ }
+
+ if (obj != -1) {
+ bitmap->avail -= cnt;
+ }
+
+ _mali_osk_spinlock_unlock(bitmap->lock);
+
+ return obj;
+}
+
+u32 _mali_osk_bitmap_avail(struct _mali_osk_bitmap *bitmap)
+{
+ MALI_DEBUG_ASSERT_POINTER(bitmap);
+
+ return bitmap->avail;
+}
+
+void _mali_osk_bitmap_free_range(struct _mali_osk_bitmap *bitmap, u32 obj, int cnt)
+{
+ MALI_DEBUG_ASSERT_POINTER(bitmap);
+
+ _mali_osk_spinlock_lock(bitmap->lock);
+ bitmap_clear(bitmap->table, obj, cnt);
+ bitmap->last = min(bitmap->last, obj);
+
+ bitmap->avail += cnt;
+ _mali_osk_spinlock_unlock(bitmap->lock);
+}
+
+int _mali_osk_bitmap_init(struct _mali_osk_bitmap *bitmap, u32 num, u32 reserve)
+{
+ MALI_DEBUG_ASSERT_POINTER(bitmap);
+ MALI_DEBUG_ASSERT(reserve <= num);
+
+ bitmap->reserve = reserve;
+ bitmap->last = reserve;
+ bitmap->max = num;
+ bitmap->avail = num - reserve;
+ bitmap->lock = _mali_osk_spinlock_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST);
+ if (!bitmap->lock) {
+ return _MALI_OSK_ERR_NOMEM;
+ }
+ bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) *
+ sizeof(long), GFP_KERNEL);
+ if (!bitmap->table) {
+ _mali_osk_spinlock_term(bitmap->lock);
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ return _MALI_OSK_ERR_OK;
+}
+
+void _mali_osk_bitmap_term(struct _mali_osk_bitmap *bitmap)
+{
+ MALI_DEBUG_ASSERT_POINTER(bitmap);
+
+ if (NULL != bitmap->lock) {
+ _mali_osk_spinlock_term(bitmap->lock);
+ }
+
+ if (NULL != bitmap->table) {
+ kfree(bitmap->table);
+ }
+}
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_irq.c b/drivers/gpu/arm/utgard/linux/mali_osk_irq.c
new file mode 100644
index 000000000000..539832d9125a
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_irq.c
@@ -0,0 +1,200 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_irq.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+
+#include <linux/slab.h> /* For memory allocation */
+#include <linux/interrupt.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+
+typedef struct _mali_osk_irq_t_struct {
+ u32 irqnum;
+ void *data;
+ _mali_osk_irq_uhandler_t uhandler;
+} mali_osk_irq_object_t;
+
+typedef irqreturn_t (*irq_handler_func_t)(int, void *, struct pt_regs *);
+static irqreturn_t irq_handler_upper_half(int port_name, void *dev_id); /* , struct pt_regs *regs*/
+
+#if defined(DEBUG)
+
+struct test_interrupt_data {
+ _mali_osk_irq_ack_t ack_func;
+ void *probe_data;
+ mali_bool interrupt_received;
+ wait_queue_head_t wq;
+};
+
+static irqreturn_t test_interrupt_upper_half(int port_name, void *dev_id)
+{
+ irqreturn_t ret = IRQ_NONE;
+ struct test_interrupt_data *data = (struct test_interrupt_data *)dev_id;
+
+ if (_MALI_OSK_ERR_OK == data->ack_func(data->probe_data)) {
+ data->interrupt_received = MALI_TRUE;
+ wake_up(&data->wq);
+ ret = IRQ_HANDLED;
+ }
+
+ return ret;
+}
+
+static _mali_osk_errcode_t test_interrupt(u32 irqnum,
+ _mali_osk_irq_trigger_t trigger_func,
+ _mali_osk_irq_ack_t ack_func,
+ void *probe_data,
+ const char *description)
+{
+ unsigned long irq_flags = 0;
+ struct test_interrupt_data data = {
+ .ack_func = ack_func,
+ .probe_data = probe_data,
+ .interrupt_received = MALI_FALSE,
+ };
+
+#if defined(CONFIG_MALI_SHARED_INTERRUPTS)
+ irq_flags |= IRQF_SHARED;
+#endif /* defined(CONFIG_MALI_SHARED_INTERRUPTS) */
+
+ if (0 != request_irq(irqnum, test_interrupt_upper_half, irq_flags, description, &data)) {
+ MALI_DEBUG_PRINT(2, ("Unable to install test IRQ handler for core '%s'\n", description));
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ init_waitqueue_head(&data.wq);
+
+ trigger_func(probe_data);
+ wait_event_timeout(data.wq, data.interrupt_received, 100);
+
+ free_irq(irqnum, &data);
+
+ if (data.interrupt_received) {
+ MALI_DEBUG_PRINT(3, ("%s: Interrupt test OK\n", description));
+ return _MALI_OSK_ERR_OK;
+ } else {
+ MALI_PRINT_ERROR(("%s: Failed interrupt test on %u\n", description, irqnum));
+ return _MALI_OSK_ERR_FAULT;
+ }
+}
+
+#endif /* defined(DEBUG) */
+
+_mali_osk_irq_t *_mali_osk_irq_init(u32 irqnum, _mali_osk_irq_uhandler_t uhandler, void *int_data, _mali_osk_irq_trigger_t trigger_func, _mali_osk_irq_ack_t ack_func, void *probe_data, const char *description)
+{
+ mali_osk_irq_object_t *irq_object;
+ unsigned long irq_flags = 0;
+
+#if defined(CONFIG_MALI_SHARED_INTERRUPTS)
+ irq_flags |= IRQF_SHARED;
+#endif /* defined(CONFIG_MALI_SHARED_INTERRUPTS) */
+
+ irq_object = kmalloc(sizeof(mali_osk_irq_object_t), GFP_KERNEL);
+ if (NULL == irq_object) {
+ return NULL;
+ }
+
+ if (-1 == irqnum) {
+ /* Probe for IRQ */
+ if ((NULL != trigger_func) && (NULL != ack_func)) {
+ unsigned long probe_count = 3;
+ _mali_osk_errcode_t err;
+ int irq;
+
+ MALI_DEBUG_PRINT(2, ("Probing for irq\n"));
+
+ do {
+ unsigned long mask;
+
+ mask = probe_irq_on();
+ trigger_func(probe_data);
+
+ _mali_osk_time_ubusydelay(5);
+
+ irq = probe_irq_off(mask);
+ err = ack_func(probe_data);
+ } while (irq < 0 && (err == _MALI_OSK_ERR_OK) && probe_count--);
+
+ if (irq < 0 || (_MALI_OSK_ERR_OK != err)) irqnum = -1;
+ else irqnum = irq;
+ } else irqnum = -1; /* no probe functions, fault */
+
+ if (-1 != irqnum) {
+ /* found an irq */
+ MALI_DEBUG_PRINT(2, ("Found irq %d\n", irqnum));
+ } else {
+ MALI_DEBUG_PRINT(2, ("Probe for irq failed\n"));
+ }
+ }
+
+ irq_object->irqnum = irqnum;
+ irq_object->uhandler = uhandler;
+ irq_object->data = int_data;
+
+ if (-1 == irqnum) {
+ MALI_DEBUG_PRINT(2, ("No IRQ for core '%s' found during probe\n", description));
+ kfree(irq_object);
+ return NULL;
+ }
+
+#if defined(DEBUG)
+ /* Verify that the configured interrupt settings are working */
+ if (_MALI_OSK_ERR_OK != test_interrupt(irqnum, trigger_func, ack_func, probe_data, description)) {
+ MALI_DEBUG_PRINT(2, ("Test of IRQ(%d) handler for core '%s' failed\n", irqnum, description));
+ kfree(irq_object);
+ return NULL;
+ }
+#endif
+
+ if (0 != request_irq(irqnum, irq_handler_upper_half, irq_flags, description, irq_object)) {
+ MALI_DEBUG_PRINT(2, ("Unable to install IRQ handler for core '%s'\n", description));
+ kfree(irq_object);
+ return NULL;
+ }
+
+ return irq_object;
+}
+
+void _mali_osk_irq_term(_mali_osk_irq_t *irq)
+{
+ mali_osk_irq_object_t *irq_object = (mali_osk_irq_object_t *)irq;
+ free_irq(irq_object->irqnum, irq_object);
+ kfree(irq_object);
+}
+
+
+/** This function is called directly in interrupt context from the OS just after
+ * the CPU get the hw-irq from mali, or other devices on the same IRQ-channel.
+ * It is registered one of these function for each mali core. When an interrupt
+ * arrives this function will be called equal times as registered mali cores.
+ * That means that we only check one mali core in one function call, and the
+ * core we check for each turn is given by the \a dev_id variable.
+ * If we detect an pending interrupt on the given core, we mask the interrupt
+ * out by settging the core's IRQ_MASK register to zero.
+ * Then we schedule the mali_core_irq_handler_bottom_half to run as high priority
+ * work queue job.
+ */
+static irqreturn_t irq_handler_upper_half(int port_name, void *dev_id) /* , struct pt_regs *regs*/
+{
+ irqreturn_t ret = IRQ_NONE;
+ mali_osk_irq_object_t *irq_object = (mali_osk_irq_object_t *)dev_id;
+
+ if (_MALI_OSK_ERR_OK == irq_object->uhandler(irq_object->data)) {
+ ret = IRQ_HANDLED;
+ }
+
+ return ret;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_locks.c b/drivers/gpu/arm/utgard/linux/mali_osk_locks.c
new file mode 100644
index 000000000000..50c0a9d23819
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_locks.c
@@ -0,0 +1,287 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_locks.c
+ * Implemenation of the OS abstraction layer for the kernel device driver
+ */
+
+#include "mali_osk_locks.h"
+#include "mali_kernel_common.h"
+#include "mali_osk.h"
+
+
+#ifdef DEBUG
+#ifdef LOCK_ORDER_CHECKING
+static DEFINE_SPINLOCK(lock_tracking_lock);
+static mali_bool add_lock_to_log_and_check(struct _mali_osk_lock_debug_s *lock, uint32_t tid);
+static void remove_lock_from_log(struct _mali_osk_lock_debug_s *lock, uint32_t tid);
+static const char *const lock_order_to_string(_mali_osk_lock_order_t order);
+#endif /* LOCK_ORDER_CHECKING */
+
+void _mali_osk_locks_debug_init(struct _mali_osk_lock_debug_s *checker, _mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order)
+{
+ checker->orig_flags = flags;
+ checker->owner = 0;
+
+#ifdef LOCK_ORDER_CHECKING
+ checker->order = order;
+ checker->next = NULL;
+#endif
+}
+
+void _mali_osk_locks_debug_add(struct _mali_osk_lock_debug_s *checker)
+{
+ checker->owner = _mali_osk_get_tid();
+
+#ifdef LOCK_ORDER_CHECKING
+ if (!(checker->orig_flags & _MALI_OSK_LOCKFLAG_UNORDERED)) {
+ if (!add_lock_to_log_and_check(checker, _mali_osk_get_tid())) {
+ printk(KERN_ERR "%d: ERROR lock %p taken while holding a lock of a higher order.\n",
+ _mali_osk_get_tid(), checker);
+ dump_stack();
+ }
+ }
+#endif
+}
+
+void _mali_osk_locks_debug_remove(struct _mali_osk_lock_debug_s *checker)
+{
+
+#ifdef LOCK_ORDER_CHECKING
+ if (!(checker->orig_flags & _MALI_OSK_LOCKFLAG_UNORDERED)) {
+ remove_lock_from_log(checker, _mali_osk_get_tid());
+ }
+#endif
+ checker->owner = 0;
+}
+
+
+#ifdef LOCK_ORDER_CHECKING
+/* Lock order checking
+ * -------------------
+ *
+ * To assure that lock ordering scheme defined by _mali_osk_lock_order_t is strictly adhered to, the
+ * following function will, together with a linked list and some extra members in _mali_osk_lock_debug_s,
+ * make sure that a lock that is taken has a higher order than the current highest-order lock a
+ * thread holds.
+ *
+ * This is done in the following manner:
+ * - A linked list keeps track of locks held by a thread.
+ * - A `next' pointer is added to each lock. This is used to chain the locks together.
+ * - When taking a lock, the `add_lock_to_log_and_check' makes sure that taking
+ * the given lock is legal. It will follow the linked list to find the last
+ * lock taken by this thread. If the last lock's order was lower than the
+ * lock that is to be taken, it appends the new lock to the list and returns
+ * true, if not, it return false. This return value is assert()'ed on in
+ * _mali_osk_lock_wait().
+ */
+
+static struct _mali_osk_lock_debug_s *lock_lookup_list;
+
+static void dump_lock_tracking_list(void)
+{
+ struct _mali_osk_lock_debug_s *l;
+ u32 n = 1;
+
+ /* print list for debugging purposes */
+ l = lock_lookup_list;
+
+ while (NULL != l) {
+ printk(" [lock: %p, tid_owner: %d, order: %d] ->", l, l->owner, l->order);
+ l = l->next;
+ MALI_DEBUG_ASSERT(n++ < 100);
+ }
+ printk(" NULL\n");
+}
+
+static int tracking_list_length(void)
+{
+ struct _mali_osk_lock_debug_s *l;
+ u32 n = 0;
+ l = lock_lookup_list;
+
+ while (NULL != l) {
+ l = l->next;
+ n++;
+ MALI_DEBUG_ASSERT(n < 100);
+ }
+ return n;
+}
+
+static mali_bool add_lock_to_log_and_check(struct _mali_osk_lock_debug_s *lock, uint32_t tid)
+{
+ mali_bool ret = MALI_FALSE;
+ _mali_osk_lock_order_t highest_order_for_tid = _MALI_OSK_LOCK_ORDER_FIRST;
+ struct _mali_osk_lock_debug_s *highest_order_lock = (struct _mali_osk_lock_debug_s *)0xbeefbabe;
+ struct _mali_osk_lock_debug_s *l;
+ unsigned long local_lock_flag;
+ u32 len;
+
+ spin_lock_irqsave(&lock_tracking_lock, local_lock_flag);
+ len = tracking_list_length();
+
+ l = lock_lookup_list;
+ if (NULL == l) { /* This is the first lock taken by this thread -- record and return true */
+ lock_lookup_list = lock;
+ spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag);
+ return MALI_TRUE;
+ } else {
+ /* Traverse the locks taken and find the lock of the highest order.
+ * Since several threads may hold locks, each lock's owner must be
+ * checked so that locks not owned by this thread can be ignored. */
+ for (;;) {
+ MALI_DEBUG_ASSERT_POINTER(l);
+ if (tid == l->owner && l->order >= highest_order_for_tid) {
+ highest_order_for_tid = l->order;
+ highest_order_lock = l;
+ }
+
+ if (NULL != l->next) {
+ l = l->next;
+ } else {
+ break;
+ }
+ }
+
+ l->next = lock;
+ l->next = NULL;
+ }
+
+ /* We have now found the highest order lock currently held by this thread and can see if it is
+ * legal to take the requested lock. */
+ ret = highest_order_for_tid < lock->order;
+
+ if (!ret) {
+ printk(KERN_ERR "Took lock of order %d (%s) while holding lock of order %d (%s)\n",
+ lock->order, lock_order_to_string(lock->order),
+ highest_order_for_tid, lock_order_to_string(highest_order_for_tid));
+ dump_lock_tracking_list();
+ }
+
+ if (len + 1 != tracking_list_length()) {
+ printk(KERN_ERR "************ lock: %p\n", lock);
+ printk(KERN_ERR "************ before: %d *** after: %d ****\n", len, tracking_list_length());
+ dump_lock_tracking_list();
+ MALI_DEBUG_ASSERT_POINTER(NULL);
+ }
+
+ spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag);
+ return ret;
+}
+
+static void remove_lock_from_log(struct _mali_osk_lock_debug_s *lock, uint32_t tid)
+{
+ struct _mali_osk_lock_debug_s *curr;
+ struct _mali_osk_lock_debug_s *prev = NULL;
+ unsigned long local_lock_flag;
+ u32 len;
+ u32 n = 0;
+
+ spin_lock_irqsave(&lock_tracking_lock, local_lock_flag);
+ len = tracking_list_length();
+ curr = lock_lookup_list;
+
+ if (NULL == curr) {
+ printk(KERN_ERR "Error: Lock tracking list was empty on call to remove_lock_from_log\n");
+ dump_lock_tracking_list();
+ }
+
+ MALI_DEBUG_ASSERT_POINTER(curr);
+
+
+ while (lock != curr) {
+ prev = curr;
+
+ MALI_DEBUG_ASSERT_POINTER(curr);
+ curr = curr->next;
+ MALI_DEBUG_ASSERT(n++ < 100);
+ }
+
+ if (NULL == prev) {
+ lock_lookup_list = curr->next;
+ } else {
+ MALI_DEBUG_ASSERT_POINTER(curr);
+ MALI_DEBUG_ASSERT_POINTER(prev);
+ prev->next = curr->next;
+ }
+
+ lock->next = NULL;
+
+ if (len - 1 != tracking_list_length()) {
+ printk(KERN_ERR "************ lock: %p\n", lock);
+ printk(KERN_ERR "************ before: %d *** after: %d ****\n", len, tracking_list_length());
+ dump_lock_tracking_list();
+ MALI_DEBUG_ASSERT_POINTER(NULL);
+ }
+
+ spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag);
+}
+
+static const char *const lock_order_to_string(_mali_osk_lock_order_t order)
+{
+ switch (order) {
+ case _MALI_OSK_LOCK_ORDER_SESSIONS:
+ return "_MALI_OSK_LOCK_ORDER_SESSIONS";
+ break;
+ case _MALI_OSK_LOCK_ORDER_MEM_SESSION:
+ return "_MALI_OSK_LOCK_ORDER_MEM_SESSION";
+ break;
+ case _MALI_OSK_LOCK_ORDER_MEM_INFO:
+ return "_MALI_OSK_LOCK_ORDER_MEM_INFO";
+ break;
+ case _MALI_OSK_LOCK_ORDER_MEM_PT_CACHE:
+ return "_MALI_OSK_LOCK_ORDER_MEM_PT_CACHE";
+ break;
+ case _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP:
+ return "_MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP";
+ break;
+ case _MALI_OSK_LOCK_ORDER_PM_EXECUTION:
+ return "_MALI_OSK_LOCK_ORDER_PM_EXECUTION";
+ break;
+ case _MALI_OSK_LOCK_ORDER_EXECUTOR:
+ return "_MALI_OSK_LOCK_ORDER_EXECUTOR";
+ break;
+ case _MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM:
+ return "_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM";
+ break;
+ case _MALI_OSK_LOCK_ORDER_SCHEDULER:
+ return "_MALI_OSK_LOCK_ORDER_SCHEDULER";
+ break;
+ case _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED:
+ return "_MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED";
+ break;
+ case _MALI_OSK_LOCK_ORDER_DMA_COMMAND:
+ return "_MALI_OSK_LOCK_ORDER_DMA_COMMAND";
+ break;
+ case _MALI_OSK_LOCK_ORDER_PROFILING:
+ return "_MALI_OSK_LOCK_ORDER_PROFILING";
+ break;
+ case _MALI_OSK_LOCK_ORDER_L2:
+ return "_MALI_OSK_LOCK_ORDER_L2";
+ break;
+ case _MALI_OSK_LOCK_ORDER_L2_COMMAND:
+ return "_MALI_OSK_LOCK_ORDER_L2_COMMAND";
+ break;
+ case _MALI_OSK_LOCK_ORDER_UTILIZATION:
+ return "_MALI_OSK_LOCK_ORDER_UTILIZATION";
+ break;
+ case _MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS:
+ return "_MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS";
+ break;
+ case _MALI_OSK_LOCK_ORDER_PM_STATE:
+ return "_MALI_OSK_LOCK_ORDER_PM_STATE";
+ break;
+ default:
+ return "<UNKNOWN_LOCK_ORDER>";
+ }
+}
+#endif /* LOCK_ORDER_CHECKING */
+#endif /* DEBUG */
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_locks.h b/drivers/gpu/arm/utgard/linux/mali_osk_locks.h
new file mode 100644
index 000000000000..aa32a81e7496
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_locks.h
@@ -0,0 +1,326 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_locks.h
+ * Defines OS abstraction of lock and mutex
+ */
+#ifndef _MALI_OSK_LOCKS_H
+#define _MALI_OSK_LOCKS_H
+
+#include <linux/spinlock.h>
+#include <linux/rwsem.h>
+#include <linux/mutex.h>
+
+#include <linux/slab.h>
+
+#include "mali_osk_types.h"
+
+#ifdef _cplusplus
+extern "C" {
+#endif
+
+ /* When DEBUG is enabled, this struct will be used to track owner, mode and order checking */
+#ifdef DEBUG
+ struct _mali_osk_lock_debug_s {
+ u32 owner;
+ _mali_osk_lock_flags_t orig_flags;
+ _mali_osk_lock_order_t order;
+ struct _mali_osk_lock_debug_s *next;
+ };
+#endif
+
+ /* Anstraction of spinlock_t */
+ struct _mali_osk_spinlock_s {
+#ifdef DEBUG
+ struct _mali_osk_lock_debug_s checker;
+#endif
+ spinlock_t spinlock;
+ };
+
+ /* Abstration of spinlock_t and lock flag which is used to store register's state before locking */
+ struct _mali_osk_spinlock_irq_s {
+#ifdef DEBUG
+ struct _mali_osk_lock_debug_s checker;
+#endif
+
+ spinlock_t spinlock;
+ unsigned long flags;
+ };
+
+ /* Abstraction of rw_semaphore in OS */
+ struct _mali_osk_mutex_rw_s {
+#ifdef DEBUG
+ struct _mali_osk_lock_debug_s checker;
+ _mali_osk_lock_mode_t mode;
+#endif
+
+ struct rw_semaphore rw_sema;
+ };
+
+ /* Mutex and mutex_interruptible functions share the same osk mutex struct */
+ struct _mali_osk_mutex_s {
+#ifdef DEBUG
+ struct _mali_osk_lock_debug_s checker;
+#endif
+ struct mutex mutex;
+ };
+
+#ifdef DEBUG
+ /** @brief _mali_osk_locks_debug_init/add/remove() functions are declared when DEBUG is enabled and
+ * defined in file mali_osk_locks.c. When LOCK_ORDER_CHECKING is enabled, calling these functions when we
+ * init/lock/unlock a lock/mutex, we could track lock order of a given tid. */
+ void _mali_osk_locks_debug_init(struct _mali_osk_lock_debug_s *checker, _mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order);
+ void _mali_osk_locks_debug_add(struct _mali_osk_lock_debug_s *checker);
+ void _mali_osk_locks_debug_remove(struct _mali_osk_lock_debug_s *checker);
+
+ /** @brief This function can return a given lock's owner when DEBUG is enabled. */
+ static inline u32 _mali_osk_lock_get_owner(struct _mali_osk_lock_debug_s *lock)
+ {
+ return lock->owner;
+ }
+#else
+#define _mali_osk_locks_debug_init(x, y, z) do {} while (0)
+#define _mali_osk_locks_debug_add(x) do {} while (0)
+#define _mali_osk_locks_debug_remove(x) do {} while (0)
+#endif
+
+ /** @brief Before use _mali_osk_spin_lock, init function should be used to allocate memory and initial spinlock*/
+ static inline _mali_osk_spinlock_t *_mali_osk_spinlock_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order)
+ {
+ _mali_osk_spinlock_t *lock = NULL;
+
+ lock = kmalloc(sizeof(_mali_osk_spinlock_t), GFP_KERNEL);
+ if (NULL == lock) {
+ return NULL;
+ }
+ spin_lock_init(&lock->spinlock);
+ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order);
+ return lock;
+ }
+
+ /** @brief Lock a spinlock */
+ static inline void _mali_osk_spinlock_lock(_mali_osk_spinlock_t *lock)
+ {
+ BUG_ON(NULL == lock);
+ spin_lock(&lock->spinlock);
+ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock);
+ }
+
+ /** @brief Unlock a spinlock */
+ static inline void _mali_osk_spinlock_unlock(_mali_osk_spinlock_t *lock)
+ {
+ BUG_ON(NULL == lock);
+ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock);
+ spin_unlock(&lock->spinlock);
+ }
+
+ /** @brief Free a memory block which the argument lock pointed to and its type must be
+ * _mali_osk_spinlock_t *. */
+ static inline void _mali_osk_spinlock_term(_mali_osk_spinlock_t *lock)
+ {
+ /* Parameter validation */
+ BUG_ON(NULL == lock);
+
+ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */
+ kfree(lock);
+ }
+
+ /** @brief Before _mali_osk_spinlock_irq_lock/unlock/term() is called, init function should be
+ * called to initial spinlock and flags in struct _mali_osk_spinlock_irq_t. */
+ static inline _mali_osk_spinlock_irq_t *_mali_osk_spinlock_irq_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order)
+ {
+ _mali_osk_spinlock_irq_t *lock = NULL;
+ lock = kmalloc(sizeof(_mali_osk_spinlock_irq_t), GFP_KERNEL);
+
+ if (NULL == lock) {
+ return NULL;
+ }
+
+ lock->flags = 0;
+ spin_lock_init(&lock->spinlock);
+ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order);
+ return lock;
+ }
+
+ /** @brief Lock spinlock and save the register's state */
+ static inline void _mali_osk_spinlock_irq_lock(_mali_osk_spinlock_irq_t *lock)
+ {
+ unsigned long tmp_flags;
+
+ BUG_ON(NULL == lock);
+ spin_lock_irqsave(&lock->spinlock, tmp_flags);
+ lock->flags = tmp_flags;
+ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock);
+ }
+
+ /** @brief Unlock spinlock with saved register's state */
+ static inline void _mali_osk_spinlock_irq_unlock(_mali_osk_spinlock_irq_t *lock)
+ {
+ BUG_ON(NULL == lock);
+ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock);
+ spin_unlock_irqrestore(&lock->spinlock, lock->flags);
+ }
+
+ /** @brief Destroy a given memory block which lock pointed to, and the lock type must be
+ * _mali_osk_spinlock_irq_t *. */
+ static inline void _mali_osk_spinlock_irq_term(_mali_osk_spinlock_irq_t *lock)
+ {
+ /* Parameter validation */
+ BUG_ON(NULL == lock);
+
+ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */
+ kfree(lock);
+ }
+
+ /** @brief Before _mali_osk_mutex_rw_wait/signal/term() is called, we should call
+ * _mali_osk_mutex_rw_init() to kmalloc a memory block and initial part of elements in it. */
+ static inline _mali_osk_mutex_rw_t *_mali_osk_mutex_rw_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order)
+ {
+ _mali_osk_mutex_rw_t *lock = NULL;
+
+ lock = kmalloc(sizeof(_mali_osk_mutex_rw_t), GFP_KERNEL);
+
+ if (NULL == lock) {
+ return NULL;
+ }
+
+ init_rwsem(&lock->rw_sema);
+ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order);
+ return lock;
+ }
+
+ /** @brief When call _mali_osk_mutex_rw_wait/signal() functions, the second argument mode
+ * should be assigned with value _MALI_OSK_LOCKMODE_RO or _MALI_OSK_LOCKMODE_RW */
+ static inline void _mali_osk_mutex_rw_wait(_mali_osk_mutex_rw_t *lock, _mali_osk_lock_mode_t mode)
+ {
+ BUG_ON(NULL == lock);
+ BUG_ON(!(_MALI_OSK_LOCKMODE_RO == mode || _MALI_OSK_LOCKMODE_RW == mode));
+
+ if (mode == _MALI_OSK_LOCKMODE_RO) {
+ down_read(&lock->rw_sema);
+ } else {
+ down_write(&lock->rw_sema);
+ }
+
+#ifdef DEBUG
+ if (mode == _MALI_OSK_LOCKMODE_RW) {
+ lock->mode = mode;
+ } else { /* mode == _MALI_OSK_LOCKMODE_RO */
+ lock->mode = mode;
+ }
+ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock);
+#endif
+ }
+
+ /** @brief Up lock->rw_sema with up_read/write() accordinf argument mode's value. */
+ static inline void _mali_osk_mutex_rw_signal(_mali_osk_mutex_rw_t *lock, _mali_osk_lock_mode_t mode)
+ {
+ BUG_ON(NULL == lock);
+ BUG_ON(!(_MALI_OSK_LOCKMODE_RO == mode || _MALI_OSK_LOCKMODE_RW == mode));
+#ifdef DEBUG
+ /* make sure the thread releasing the lock actually was the owner */
+ if (mode == _MALI_OSK_LOCKMODE_RW) {
+ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock);
+ /* This lock now has no owner */
+ lock->checker.owner = 0;
+ }
+#endif
+
+ if (mode == _MALI_OSK_LOCKMODE_RO) {
+ up_read(&lock->rw_sema);
+ } else {
+ up_write(&lock->rw_sema);
+ }
+ }
+
+ /** @brief Free a given memory block which lock pointed to and its type must be
+ * _mali_sok_mutex_rw_t *. */
+ static inline void _mali_osk_mutex_rw_term(_mali_osk_mutex_rw_t *lock)
+ {
+ /* Parameter validation */
+ BUG_ON(NULL == lock);
+
+ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */
+ kfree(lock);
+ }
+
+ /** @brief Mutex & mutex_interruptible share the same init and term function, because they have the
+ * same osk mutex struct, and the difference between them is which locking function they use */
+ static inline _mali_osk_mutex_t *_mali_osk_mutex_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order)
+ {
+ _mali_osk_mutex_t *lock = NULL;
+
+ lock = kmalloc(sizeof(_mali_osk_mutex_t), GFP_KERNEL);
+
+ if (NULL == lock) {
+ return NULL;
+ }
+ mutex_init(&lock->mutex);
+
+ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order);
+ return lock;
+ }
+
+ /** @brief Lock the lock->mutex with mutex_lock_interruptible function */
+ static inline _mali_osk_errcode_t _mali_osk_mutex_wait_interruptible(_mali_osk_mutex_t *lock)
+ {
+ _mali_osk_errcode_t err = _MALI_OSK_ERR_OK;
+
+ BUG_ON(NULL == lock);
+
+ if (mutex_lock_interruptible(&lock->mutex)) {
+ printk(KERN_WARNING "Mali: Can not lock mutex\n");
+ err = _MALI_OSK_ERR_RESTARTSYSCALL;
+ }
+
+ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock);
+ return err;
+ }
+
+ /** @brief Unlock the lock->mutex which is locked with mutex_lock_interruptible() function. */
+ static inline void _mali_osk_mutex_signal_interruptible(_mali_osk_mutex_t *lock)
+ {
+ BUG_ON(NULL == lock);
+ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock);
+ mutex_unlock(&lock->mutex);
+ }
+
+ /** @brief Lock the lock->mutex just with mutex_lock() function which could not be interruptted. */
+ static inline void _mali_osk_mutex_wait(_mali_osk_mutex_t *lock)
+ {
+ BUG_ON(NULL == lock);
+ mutex_lock(&lock->mutex);
+ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock);
+ }
+
+ /** @brief Unlock the lock->mutex which is locked with mutex_lock() function. */
+ static inline void _mali_osk_mutex_signal(_mali_osk_mutex_t *lock)
+ {
+ BUG_ON(NULL == lock);
+ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock);
+ mutex_unlock(&lock->mutex);
+ }
+
+ /** @brief Free a given memory block which lock point. */
+ static inline void _mali_osk_mutex_term(_mali_osk_mutex_t *lock)
+ {
+ /* Parameter validation */
+ BUG_ON(NULL == lock);
+
+ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */
+ kfree(lock);
+ }
+
+#ifdef _cplusplus
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_low_level_mem.c b/drivers/gpu/arm/utgard/linux/mali_osk_low_level_mem.c
new file mode 100644
index 000000000000..c14e872a7d63
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_low_level_mem.c
@@ -0,0 +1,146 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_low_level_mem.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+
+#include <asm/io.h>
+#include <linux/ioport.h>
+#include <linux/slab.h>
+
+#include "mali_kernel_common.h"
+#include "mali_osk.h"
+#include "mali_ukk.h"
+
+void _mali_osk_mem_barrier(void)
+{
+ mb();
+}
+
+void _mali_osk_write_mem_barrier(void)
+{
+ wmb();
+}
+
+mali_io_address _mali_osk_mem_mapioregion(uintptr_t phys, u32 size, const char *description)
+{
+ return (mali_io_address)ioremap_nocache(phys, size);
+}
+
+void _mali_osk_mem_unmapioregion(uintptr_t phys, u32 size, mali_io_address virt)
+{
+ iounmap((void *)virt);
+}
+
+_mali_osk_errcode_t inline _mali_osk_mem_reqregion(uintptr_t phys, u32 size, const char *description)
+{
+#if MALI_LICENSE_IS_GPL
+ return _MALI_OSK_ERR_OK; /* GPL driver gets the mem region for the resources registered automatically */
+#else
+ return ((NULL == request_mem_region(phys, size, description)) ? _MALI_OSK_ERR_NOMEM : _MALI_OSK_ERR_OK);
+#endif
+}
+
+void inline _mali_osk_mem_unreqregion(uintptr_t phys, u32 size)
+{
+#if !MALI_LICENSE_IS_GPL
+ release_mem_region(phys, size);
+#endif
+}
+
+void inline _mali_osk_mem_iowrite32_relaxed(volatile mali_io_address addr, u32 offset, u32 val)
+{
+ __raw_writel(cpu_to_le32(val), ((u8 *)addr) + offset);
+}
+
+u32 inline _mali_osk_mem_ioread32(volatile mali_io_address addr, u32 offset)
+{
+ return ioread32(((u8 *)addr) + offset);
+}
+
+void inline _mali_osk_mem_iowrite32(volatile mali_io_address addr, u32 offset, u32 val)
+{
+ iowrite32(val, ((u8 *)addr) + offset);
+}
+
+void _mali_osk_cache_flushall(void)
+{
+ /** @note Cached memory is not currently supported in this implementation */
+}
+
+void _mali_osk_cache_ensure_uncached_range_flushed(void *uncached_mapping, u32 offset, u32 size)
+{
+ _mali_osk_write_mem_barrier();
+}
+
+u32 _mali_osk_mem_write_safe(void __user *dest, const void __user *src, u32 size)
+{
+#define MALI_MEM_SAFE_COPY_BLOCK_SIZE 4096
+ u32 retval = 0;
+ void *temp_buf;
+
+ temp_buf = kmalloc(MALI_MEM_SAFE_COPY_BLOCK_SIZE, GFP_KERNEL);
+ if (NULL != temp_buf) {
+ u32 bytes_left_to_copy = size;
+ u32 i;
+ for (i = 0; i < size; i += MALI_MEM_SAFE_COPY_BLOCK_SIZE) {
+ u32 size_to_copy;
+ u32 size_copied;
+ u32 bytes_left;
+
+ if (bytes_left_to_copy > MALI_MEM_SAFE_COPY_BLOCK_SIZE) {
+ size_to_copy = MALI_MEM_SAFE_COPY_BLOCK_SIZE;
+ } else {
+ size_to_copy = bytes_left_to_copy;
+ }
+
+ bytes_left = copy_from_user(temp_buf, ((char *)src) + i, size_to_copy);
+ size_copied = size_to_copy - bytes_left;
+
+ bytes_left = copy_to_user(((char *)dest) + i, temp_buf, size_copied);
+ size_copied -= bytes_left;
+
+ bytes_left_to_copy -= size_copied;
+ retval += size_copied;
+
+ if (size_copied != size_to_copy) {
+ break; /* Early out, we was not able to copy this entire block */
+ }
+ }
+
+ kfree(temp_buf);
+ }
+
+ return retval;
+}
+
+_mali_osk_errcode_t _mali_ukk_mem_write_safe(_mali_uk_mem_write_safe_s *args)
+{
+ void __user *src;
+ void __user *dst;
+ struct mali_session_data *session;
+
+ MALI_DEBUG_ASSERT_POINTER(args);
+
+ session = (struct mali_session_data *)(uintptr_t)args->ctx;
+
+ if (NULL == session) {
+ return _MALI_OSK_ERR_INVALID_ARGS;
+ }
+
+ src = (void __user *)(uintptr_t)args->src;
+ dst = (void __user *)(uintptr_t)args->dest;
+
+ /* Return number of bytes actually copied */
+ args->size = _mali_osk_mem_write_safe(dst, src, args->size);
+ return _MALI_OSK_ERR_OK;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_mali.c b/drivers/gpu/arm/utgard/linux/mali_osk_mali.c
new file mode 100644
index 000000000000..a3749d8057b4
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_mali.c
@@ -0,0 +1,390 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_mali.c
+ * Implementation of the OS abstraction layer which is specific for the Mali kernel device driver
+ */
+#include <linux/kernel.h>
+#include <asm/uaccess.h>
+#include <linux/platform_device.h>
+#include <linux/mali/mali_utgard.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+
+#include "mali_osk_mali.h"
+#include "mali_kernel_common.h" /* MALI_xxx macros */
+#include "mali_osk.h" /* kernel side OS functions */
+#include "mali_kernel_linux.h"
+
+
+#if defined(CONFIG_MALI_DT) && !defined(CONFIG_MALI_PLAT_SPECIFIC_DT)
+
+#define MALI_OSK_INVALID_RESOURCE_ADDRESS 0xFFFFFFFF
+
+/**
+ * Define the max number of resource we could have.
+ */
+#define MALI_OSK_MAX_RESOURCE_NUMBER 27
+
+/**
+ * Define the max number of resource with interrupts, and they are
+ * the first 20 elements in array mali_osk_resource_bank.
+ */
+#define MALI_OSK_RESOURCE_WITH_IRQ_NUMBER 20
+
+/**
+ * pp core start and end location in mali_osk_resource_bank array.
+ */
+#define MALI_OSK_RESOURCE_PP_LOCATION_START 2
+#define MALI_OSK_RESOURCE_PP_LOCATION_END 17
+
+/**
+ * L2 cache start and end location in mali_osk_resource_bank array.
+ */
+#define MALI_OSK_RESOURCE_L2_LOCATION_START 20
+#define MALI_OSK_RESOURCE_l2_LOCATION_END 22
+
+/**
+ * DMA unit location.
+ */
+#define MALI_OSK_RESOURCE_DMA_LOCATION 26
+
+static _mali_osk_resource_t mali_osk_resource_bank[MALI_OSK_MAX_RESOURCE_NUMBER] = {
+ {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "IRQGP",},
+ {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "IRQGPMMU",},
+ {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "IRQPP0",},
+ {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "IRQPPMMU0",},
+ {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "IRQPP1",},
+ {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "IRQPPMMU1",},
+ {.description = "Mali_PP2", .base = MALI_OFFSET_PP2, .irq_name = "IRQPP2",},
+ {.description = "Mali_PP2_MMU", .base = MALI_OFFSET_PP2_MMU, .irq_name = "IRQPPMMU2",},
+ {.description = "Mali_PP3", .base = MALI_OFFSET_PP3, .irq_name = "IRQPP3",},
+ {.description = "Mali_PP3_MMU", .base = MALI_OFFSET_PP3_MMU, .irq_name = "IRQPPMMU3",},
+ {.description = "Mali_PP4", .base = MALI_OFFSET_PP4, .irq_name = "IRQPP4",},
+ {.description = "Mali_PP4_MMU", .base = MALI_OFFSET_PP4_MMU, .irq_name = "IRQPPMMU4",},
+ {.description = "Mali_PP5", .base = MALI_OFFSET_PP5, .irq_name = "IRQPP5",},
+ {.description = "Mali_PP5_MMU", .base = MALI_OFFSET_PP5_MMU, .irq_name = "IRQPPMMU5",},
+ {.description = "Mali_PP6", .base = MALI_OFFSET_PP6, .irq_name = "IRQPP6",},
+ {.description = "Mali_PP6_MMU", .base = MALI_OFFSET_PP6_MMU, .irq_name = "IRQPPMMU6",},
+ {.description = "Mali_PP7", .base = MALI_OFFSET_PP7, .irq_name = "IRQPP7",},
+ {.description = "Mali_PP7_MMU", .base = MALI_OFFSET_PP7_MMU, .irq_name = "IRQPPMMU",},
+ {.description = "Mali_PP_Broadcast", .base = MALI_OFFSET_PP_BCAST, .irq_name = "IRQPP",},
+ {.description = "Mali_PMU", .base = MALI_OFFSET_PMU, .irq_name = "IRQPMU",},
+ {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE0,},
+ {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE1,},
+ {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE2,},
+ {.description = "Mali_PP_MMU_Broadcast", .base = MALI_OFFSET_PP_BCAST_MMU,},
+ {.description = "Mali_Broadcast", .base = MALI_OFFSET_BCAST,},
+ {.description = "Mali_DLBU", .base = MALI_OFFSET_DLBU,},
+ {.description = "Mali_DMA", .base = MALI_OFFSET_DMA,},
+};
+
+static int _mali_osk_get_compatible_name(const char **out_string)
+{
+ struct device_node *node = mali_platform_device->dev.of_node;
+
+ MALI_DEBUG_ASSERT(NULL != node);
+
+ return of_property_read_string(node, "compatible", out_string);
+}
+
+_mali_osk_errcode_t _mali_osk_resource_initialize(void)
+{
+ mali_bool mali_is_450 = MALI_FALSE, mali_is_470 = MALI_FALSE;
+ int i, pp_core_num = 0, l2_core_num = 0;
+ struct resource *res;
+ const char *compatible_name = NULL;
+
+ if (0 == _mali_osk_get_compatible_name(&compatible_name)) {
+ if (0 == strncmp(compatible_name, "arm,mali-450", strlen("arm,mali-450"))) {
+ mali_is_450 = MALI_TRUE;
+ MALI_DEBUG_PRINT(2, ("mali-450 device tree detected."));
+ } else if (0 == strncmp(compatible_name, "arm,mali-470", strlen("arm,mali-470"))) {
+ mali_is_470 = MALI_TRUE;
+ MALI_DEBUG_PRINT(2, ("mali-470 device tree detected."));
+ }
+ }
+
+ for (i = 0; i < MALI_OSK_RESOURCE_WITH_IRQ_NUMBER; i++) {
+ res = platform_get_resource_byname(mali_platform_device, IORESOURCE_IRQ, mali_osk_resource_bank[i].irq_name);
+ if (res) {
+ mali_osk_resource_bank[i].irq = res->start;
+ } else {
+ mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS;
+ }
+ }
+
+ for (i = MALI_OSK_RESOURCE_PP_LOCATION_START; i <= MALI_OSK_RESOURCE_PP_LOCATION_END; i++) {
+ if (MALI_OSK_INVALID_RESOURCE_ADDRESS != mali_osk_resource_bank[i].base) {
+ pp_core_num++;
+ }
+ }
+
+ /* We have to divide by 2, because we caculate twice for only one pp(pp_core and pp_mmu_core). */
+ if (0 != pp_core_num % 2) {
+ MALI_DEBUG_PRINT(2, ("The value of pp core number isn't normal."));
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ pp_core_num /= 2;
+
+ /**
+ * we can caculate the number of l2 cache core according the number of pp core number
+ * and device type(mali400/mali450/mali470).
+ */
+ l2_core_num = 1;
+ if (mali_is_450) {
+ if (pp_core_num > 4) {
+ l2_core_num = 3;
+ } else if (pp_core_num <= 4) {
+ l2_core_num = 2;
+ }
+ }
+
+ for (i = MALI_OSK_RESOURCE_l2_LOCATION_END; i > MALI_OSK_RESOURCE_L2_LOCATION_START + l2_core_num - 1; i--) {
+ mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS;
+ }
+
+ /* If device is not mali-450 type, we have to remove related resource from resource bank. */
+ if (!(mali_is_450 || mali_is_470)) {
+ for (i = MALI_OSK_RESOURCE_l2_LOCATION_END + 1; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) {
+ mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS;
+ }
+ }
+
+ if (mali_is_470)
+ mali_osk_resource_bank[MALI_OSK_RESOURCE_DMA_LOCATION].base = MALI_OSK_INVALID_RESOURCE_ADDRESS;
+
+ return _MALI_OSK_ERR_OK;
+}
+
+_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res)
+{
+ int i;
+
+ if (NULL == mali_platform_device) {
+ return _MALI_OSK_ERR_ITEM_NOT_FOUND;
+ }
+
+ /* Traverse all of resources in resources bank to find the matching one. */
+ for (i = 0; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) {
+ if (mali_osk_resource_bank[i].base == addr) {
+ if (NULL != res) {
+ res->base = addr + _mali_osk_resource_base_address();
+ res->description = mali_osk_resource_bank[i].description;
+ res->irq = mali_osk_resource_bank[i].irq;
+ }
+ return _MALI_OSK_ERR_OK;
+ }
+ }
+
+ return _MALI_OSK_ERR_ITEM_NOT_FOUND;
+}
+
+uintptr_t _mali_osk_resource_base_address(void)
+{
+ struct resource *reg_res = NULL;
+ uintptr_t ret = 0;
+
+ reg_res = platform_get_resource(mali_platform_device, IORESOURCE_MEM, 0);
+
+ if (NULL != reg_res) {
+ ret = reg_res->start;
+ }
+
+ return ret;
+}
+
+void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size)
+{
+ struct device_node *node = mali_platform_device->dev.of_node;
+ struct property *prop;
+ const __be32 *p;
+ int length = 0, i = 0;
+ u32 u;
+
+ MALI_DEBUG_PRINT(2, ("Get pmu config from device tree configuration.\n"));
+
+ MALI_DEBUG_ASSERT(NULL != node);
+
+ if (!of_get_property(node, "pmu_domain_config", &length)) {
+ return;
+ }
+
+ if (array_size != length / sizeof(u32)) {
+ MALI_PRINT_ERROR(("Wrong pmu domain config in device tree."));
+ return;
+ }
+
+ of_property_for_each_u32(node, "pmu_domain_config", prop, p, u) {
+ domain_config_array[i] = (u16)u;
+ i++;
+ }
+
+ return;
+}
+
+u32 _mali_osk_get_pmu_switch_delay(void)
+{
+ struct device_node *node = mali_platform_device->dev.of_node;
+ u32 switch_delay;
+
+ MALI_DEBUG_ASSERT(NULL != node);
+
+ if (0 == of_property_read_u32(node, "pmu_switch_delay", &switch_delay)) {
+ return switch_delay;
+ } else {
+ MALI_DEBUG_PRINT(2, ("Couldn't find pmu_switch_delay in device tree configuration.\n"));
+ }
+
+ return 0;
+}
+
+#else /* CONFIG_MALI_DT && !CONFIG_MALI_PLAT_SPECIFIC_DT */
+
+_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res)
+{
+ int i;
+ uintptr_t phys_addr;
+
+ if (NULL == mali_platform_device) {
+ /* Not connected to a device */
+ return _MALI_OSK_ERR_ITEM_NOT_FOUND;
+ }
+
+ phys_addr = addr + _mali_osk_resource_base_address();
+ for (i = 0; i < mali_platform_device->num_resources; i++) {
+ if (IORESOURCE_MEM == resource_type(&(mali_platform_device->resource[i])) &&
+ mali_platform_device->resource[i].start == phys_addr) {
+ if (NULL != res) {
+ res->base = phys_addr;
+ res->description = mali_platform_device->resource[i].name;
+
+ /* Any (optional) IRQ resource belonging to this resource will follow */
+ if ((i + 1) < mali_platform_device->num_resources &&
+ IORESOURCE_IRQ == resource_type(&(mali_platform_device->resource[i + 1]))) {
+ res->irq = mali_platform_device->resource[i + 1].start;
+ } else {
+ res->irq = -1;
+ }
+ }
+ return _MALI_OSK_ERR_OK;
+ }
+ }
+
+ return _MALI_OSK_ERR_ITEM_NOT_FOUND;
+}
+
+uintptr_t _mali_osk_resource_base_address(void)
+{
+ uintptr_t lowest_addr = (uintptr_t)(0 - 1);
+ uintptr_t ret = 0;
+
+ if (NULL != mali_platform_device) {
+ int i;
+ for (i = 0; i < mali_platform_device->num_resources; i++) {
+ if (mali_platform_device->resource[i].flags & IORESOURCE_MEM &&
+ mali_platform_device->resource[i].start < lowest_addr) {
+ lowest_addr = mali_platform_device->resource[i].start;
+ ret = lowest_addr;
+ }
+ }
+ }
+
+ return ret;
+}
+
+void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size)
+{
+ _mali_osk_device_data data = { 0, };
+
+ MALI_DEBUG_PRINT(2, ("Get pmu config from platform device data.\n"));
+ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) {
+ /* Copy the custom customer power domain config */
+ _mali_osk_memcpy(domain_config_array, data.pmu_domain_config, sizeof(data.pmu_domain_config));
+ }
+
+ return;
+}
+
+u32 _mali_osk_get_pmu_switch_delay(void)
+{
+ _mali_osk_errcode_t err;
+ _mali_osk_device_data data = { 0, };
+
+ err = _mali_osk_device_data_get(&data);
+
+ if (_MALI_OSK_ERR_OK == err) {
+ return data.pmu_switch_delay;
+ }
+
+ return 0;
+}
+#endif /* CONFIG_MALI_DT */
+
+_mali_osk_errcode_t _mali_osk_device_data_get(_mali_osk_device_data *data)
+{
+ MALI_DEBUG_ASSERT_POINTER(data);
+
+ if (NULL != mali_platform_device) {
+ struct mali_gpu_device_data *os_data = NULL;
+
+ os_data = (struct mali_gpu_device_data *)mali_platform_device->dev.platform_data;
+ if (NULL != os_data) {
+ /* Copy data from OS dependant struct to Mali neutral struct (identical!) */
+ BUILD_BUG_ON(sizeof(*os_data) != sizeof(*data));
+ _mali_osk_memcpy(data, os_data, sizeof(*os_data));
+
+ return _MALI_OSK_ERR_OK;
+ }
+ }
+
+ return _MALI_OSK_ERR_ITEM_NOT_FOUND;
+}
+
+u32 _mali_osk_identify_gpu_resource(void)
+{
+ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_L2_RESOURCE1, NULL))
+ /* Mali 450 */
+ return 0x450;
+
+ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_DLBU, NULL))
+ /* Mali 470 */
+ return 0x470;
+
+ /* Mali 400 */
+ return 0x400;
+}
+
+mali_bool _mali_osk_shared_interrupts(void)
+{
+ u32 irqs[128];
+ u32 i, j, irq, num_irqs_found = 0;
+
+ MALI_DEBUG_ASSERT_POINTER(mali_platform_device);
+ MALI_DEBUG_ASSERT(128 >= mali_platform_device->num_resources);
+
+ for (i = 0; i < mali_platform_device->num_resources; i++) {
+ if (IORESOURCE_IRQ & mali_platform_device->resource[i].flags) {
+ irq = mali_platform_device->resource[i].start;
+
+ for (j = 0; j < num_irqs_found; ++j) {
+ if (irq == irqs[j]) {
+ return MALI_TRUE;
+ }
+ }
+
+ irqs[num_irqs_found++] = irq;
+ }
+ }
+
+ return MALI_FALSE;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_math.c b/drivers/gpu/arm/utgard/linux/mali_osk_math.c
new file mode 100644
index 000000000000..085ce76f7665
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_math.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_math.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+
+#include "mali_osk.h"
+#include <linux/bitops.h>
+
+u32 _mali_osk_clz(u32 input)
+{
+ return 32 - fls(input);
+}
+
+u32 _mali_osk_fls(u32 input)
+{
+ return fls(input);
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_memory.c b/drivers/gpu/arm/utgard/linux/mali_osk_memory.c
new file mode 100644
index 000000000000..390e613e186d
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_memory.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_memory.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+
+#include "mali_osk.h"
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+void inline *_mali_osk_calloc(u32 n, u32 size)
+{
+ return kcalloc(n, size, GFP_KERNEL);
+}
+
+void inline *_mali_osk_malloc(u32 size)
+{
+ return kmalloc(size, GFP_KERNEL);
+}
+
+void inline _mali_osk_free(void *ptr)
+{
+ kfree(ptr);
+}
+
+void inline *_mali_osk_valloc(u32 size)
+{
+ return vmalloc(size);
+}
+
+void inline _mali_osk_vfree(void *ptr)
+{
+ vfree(ptr);
+}
+
+void inline *_mali_osk_memcpy(void *dst, const void *src, u32 len)
+{
+ return memcpy(dst, src, len);
+}
+
+void inline *_mali_osk_memset(void *s, u32 c, u32 n)
+{
+ return memset(s, c, n);
+}
+
+mali_bool _mali_osk_mem_check_allocated(u32 max_allocated)
+{
+ /* No need to prevent an out-of-memory dialogue appearing on Linux,
+ * so we always return MALI_TRUE.
+ */
+ return MALI_TRUE;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_misc.c b/drivers/gpu/arm/utgard/linux/mali_osk_misc.c
new file mode 100644
index 000000000000..0a619e3fc27e
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_misc.c
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_misc.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+#include <linux/kernel.h>
+#include <asm/uaccess.h>
+#include <asm/cacheflush.h>
+#include <linux/sched.h>
+#include <linux/seq_file.h>
+#include <linux/module.h>
+#include "mali_osk.h"
+
+#if !defined(CONFIG_MALI_QUIET)
+void _mali_osk_dbgmsg(const char *fmt, ...)
+{
+ va_list args;
+ va_start(args, fmt);
+ vprintk(fmt, args);
+ va_end(args);
+}
+#endif /* !defined(CONFIG_MALI_QUIET) */
+
+u32 _mali_osk_snprintf(char *buf, u32 size, const char *fmt, ...)
+{
+ int res;
+ va_list args;
+ va_start(args, fmt);
+
+ res = vscnprintf(buf, (size_t)size, fmt, args);
+
+ va_end(args);
+ return res;
+}
+
+void _mali_osk_ctxprintf(_mali_osk_print_ctx *print_ctx, const char *fmt, ...)
+{
+ va_list args;
+ char buf[512];
+
+ va_start(args, fmt);
+ vscnprintf(buf, 512, fmt, args);
+ seq_printf(print_ctx, buf);
+ va_end(args);
+}
+
+void _mali_osk_abort(void)
+{
+ /* make a simple fault by dereferencing a NULL pointer */
+ dump_stack();
+ *(int *)0 = 0;
+}
+
+void _mali_osk_break(void)
+{
+ _mali_osk_abort();
+}
+
+u32 _mali_osk_get_pid(void)
+{
+ /* Thread group ID is the process ID on Linux */
+ return (u32)current->tgid;
+}
+
+char *_mali_osk_get_comm(void)
+{
+ return (char *)current->comm;
+}
+
+
+u32 _mali_osk_get_tid(void)
+{
+ /* pid is actually identifying the thread on Linux */
+ u32 tid = current->pid;
+
+ /* If the pid is 0 the core was idle. Instead of returning 0 we return a special number
+ * identifying which core we are on. */
+ if (0 == tid) {
+ tid = -(1 + raw_smp_processor_id());
+ }
+
+ return tid;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_notification.c b/drivers/gpu/arm/utgard/linux/mali_osk_notification.c
new file mode 100644
index 000000000000..e66fe83f3557
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_notification.c
@@ -0,0 +1,182 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_notification.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+/**
+ * Declaration of the notification queue object type
+ * Contains a linked list of notification pending delivery to user space.
+ * It also contains a wait queue of exclusive waiters blocked in the ioctl
+ * When a new notification is posted a single thread is resumed.
+ */
+struct _mali_osk_notification_queue_t_struct {
+ spinlock_t mutex; /**< Mutex protecting the list */
+ wait_queue_head_t receive_queue; /**< Threads waiting for new entries to the queue */
+ struct list_head head; /**< List of notifications waiting to be picked up */
+};
+
+typedef struct _mali_osk_notification_wrapper_t_struct {
+ struct list_head list; /**< Internal linked list variable */
+ _mali_osk_notification_t data; /**< Notification data */
+} _mali_osk_notification_wrapper_t;
+
+_mali_osk_notification_queue_t *_mali_osk_notification_queue_init(void)
+{
+ _mali_osk_notification_queue_t *result;
+
+ result = (_mali_osk_notification_queue_t *)kmalloc(sizeof(_mali_osk_notification_queue_t), GFP_KERNEL);
+ if (NULL == result) return NULL;
+
+ spin_lock_init(&result->mutex);
+ init_waitqueue_head(&result->receive_queue);
+ INIT_LIST_HEAD(&result->head);
+
+ return result;
+}
+
+_mali_osk_notification_t *_mali_osk_notification_create(u32 type, u32 size)
+{
+ /* OPT Recycling of notification objects */
+ _mali_osk_notification_wrapper_t *notification;
+
+ notification = (_mali_osk_notification_wrapper_t *)kmalloc(sizeof(_mali_osk_notification_wrapper_t) + size,
+ GFP_KERNEL | __GFP_HIGH | __GFP_REPEAT);
+ if (NULL == notification) {
+ MALI_DEBUG_PRINT(1, ("Failed to create a notification object\n"));
+ return NULL;
+ }
+
+ /* Init the list */
+ INIT_LIST_HEAD(&notification->list);
+
+ if (0 != size) {
+ notification->data.result_buffer = ((u8 *)notification) + sizeof(_mali_osk_notification_wrapper_t);
+ } else {
+ notification->data.result_buffer = NULL;
+ }
+
+ /* set up the non-allocating fields */
+ notification->data.notification_type = type;
+ notification->data.result_buffer_size = size;
+
+ /* all ok */
+ return &(notification->data);
+}
+
+void _mali_osk_notification_delete(_mali_osk_notification_t *object)
+{
+ _mali_osk_notification_wrapper_t *notification;
+ MALI_DEBUG_ASSERT_POINTER(object);
+
+ notification = container_of(object, _mali_osk_notification_wrapper_t, data);
+
+ /* Free the container */
+ kfree(notification);
+}
+
+void _mali_osk_notification_queue_term(_mali_osk_notification_queue_t *queue)
+{
+ _mali_osk_notification_t *result;
+ MALI_DEBUG_ASSERT_POINTER(queue);
+
+ while (_MALI_OSK_ERR_OK == _mali_osk_notification_queue_dequeue(queue, &result)) {
+ _mali_osk_notification_delete(result);
+ }
+
+ /* not much to do, just free the memory */
+ kfree(queue);
+}
+void _mali_osk_notification_queue_send(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t *object)
+{
+#if defined(MALI_UPPER_HALF_SCHEDULING)
+ unsigned long irq_flags;
+#endif
+
+ _mali_osk_notification_wrapper_t *notification;
+ MALI_DEBUG_ASSERT_POINTER(queue);
+ MALI_DEBUG_ASSERT_POINTER(object);
+
+ notification = container_of(object, _mali_osk_notification_wrapper_t, data);
+
+#if defined(MALI_UPPER_HALF_SCHEDULING)
+ spin_lock_irqsave(&queue->mutex, irq_flags);
+#else
+ spin_lock(&queue->mutex);
+#endif
+
+ list_add_tail(&notification->list, &queue->head);
+
+#if defined(MALI_UPPER_HALF_SCHEDULING)
+ spin_unlock_irqrestore(&queue->mutex, irq_flags);
+#else
+ spin_unlock(&queue->mutex);
+#endif
+
+ /* and wake up one possible exclusive waiter */
+ wake_up(&queue->receive_queue);
+}
+
+_mali_osk_errcode_t _mali_osk_notification_queue_dequeue(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result)
+{
+#if defined(MALI_UPPER_HALF_SCHEDULING)
+ unsigned long irq_flags;
+#endif
+
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_ITEM_NOT_FOUND;
+ _mali_osk_notification_wrapper_t *wrapper_object;
+
+#if defined(MALI_UPPER_HALF_SCHEDULING)
+ spin_lock_irqsave(&queue->mutex, irq_flags);
+#else
+ spin_lock(&queue->mutex);
+#endif
+
+ if (!list_empty(&queue->head)) {
+ wrapper_object = list_entry(queue->head.next, _mali_osk_notification_wrapper_t, list);
+ *result = &(wrapper_object->data);
+ list_del_init(&wrapper_object->list);
+ ret = _MALI_OSK_ERR_OK;
+ }
+
+#if defined(MALI_UPPER_HALF_SCHEDULING)
+ spin_unlock_irqrestore(&queue->mutex, irq_flags);
+#else
+ spin_unlock(&queue->mutex);
+#endif
+
+ return ret;
+}
+
+_mali_osk_errcode_t _mali_osk_notification_queue_receive(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result)
+{
+ /* check input */
+ MALI_DEBUG_ASSERT_POINTER(queue);
+ MALI_DEBUG_ASSERT_POINTER(result);
+
+ /* default result */
+ *result = NULL;
+
+ if (wait_event_interruptible(queue->receive_queue,
+ _MALI_OSK_ERR_OK == _mali_osk_notification_queue_dequeue(queue, result))) {
+ return _MALI_OSK_ERR_RESTARTSYSCALL;
+ }
+
+ return _MALI_OSK_ERR_OK; /* all ok */
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_pm.c b/drivers/gpu/arm/utgard/linux/mali_osk_pm.c
new file mode 100644
index 000000000000..21180d33fe75
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_pm.c
@@ -0,0 +1,83 @@
+/**
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_pm.c
+ * Implementation of the callback functions from common power management
+ */
+
+#include <linux/sched.h>
+
+#ifdef CONFIG_PM_RUNTIME
+#include <linux/pm_runtime.h>
+#endif /* CONFIG_PM_RUNTIME */
+#include <linux/platform_device.h>
+#include <linux/version.h>
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_kernel_linux.h"
+
+/* Can NOT run in atomic context */
+_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_sync(void)
+{
+#ifdef CONFIG_PM_RUNTIME
+ int err;
+ MALI_DEBUG_ASSERT_POINTER(mali_platform_device);
+ err = pm_runtime_get_sync(&(mali_platform_device->dev));
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+ pm_runtime_mark_last_busy(&(mali_platform_device->dev));
+#endif
+ if (0 > err) {
+ MALI_PRINT_ERROR(("Mali OSK PM: pm_runtime_get_sync() returned error code %d\n", err));
+ return _MALI_OSK_ERR_FAULT;
+ }
+#endif
+ return _MALI_OSK_ERR_OK;
+}
+
+/* Can run in atomic context */
+_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_async(void)
+{
+#ifdef CONFIG_PM_RUNTIME
+ int err;
+ MALI_DEBUG_ASSERT_POINTER(mali_platform_device);
+ err = pm_runtime_get(&(mali_platform_device->dev));
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+ pm_runtime_mark_last_busy(&(mali_platform_device->dev));
+#endif
+ if (0 > err && -EINPROGRESS != err) {
+ MALI_PRINT_ERROR(("Mali OSK PM: pm_runtime_get() returned error code %d\n", err));
+ return _MALI_OSK_ERR_FAULT;
+ }
+#endif
+ return _MALI_OSK_ERR_OK;
+}
+
+
+/* Can run in atomic context */
+void _mali_osk_pm_dev_ref_put(void)
+{
+#ifdef CONFIG_PM_RUNTIME
+ MALI_DEBUG_ASSERT_POINTER(mali_platform_device);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+ pm_runtime_mark_last_busy(&(mali_platform_device->dev));
+ pm_runtime_put_autosuspend(&(mali_platform_device->dev));
+#else
+ pm_runtime_put(&(mali_platform_device->dev));
+#endif
+#endif
+}
+
+void _mali_osk_pm_dev_barrier(void)
+{
+#ifdef CONFIG_PM_RUNTIME
+ pm_runtime_barrier(&(mali_platform_device->dev));
+#endif
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_profiling.c b/drivers/gpu/arm/utgard/linux/mali_osk_profiling.c
new file mode 100644
index 000000000000..cc09748e7316
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_profiling.c
@@ -0,0 +1,1272 @@
+/*
+ * Copyright (C) 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/hrtimer.h>
+#include <linux/module.h>
+#include <linux/file.h>
+#include <linux/poll.h>
+#include <linux/anon_inodes.h>
+#include <linux/sched.h>
+
+#include <mali_profiling_gator_api.h>
+#include "mali_kernel_common.h"
+#include "mali_osk.h"
+#include "mali_ukk.h"
+#include "mali_uk_types.h"
+#include "mali_osk_profiling.h"
+#include "mali_linux_trace.h"
+#include "mali_gp.h"
+#include "mali_pp.h"
+#include "mali_l2_cache.h"
+#include "mali_user_settings_db.h"
+#include "mali_executor.h"
+#include "mali_memory_manager.h"
+
+#define MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE 100
+#define MALI_PROFILING_STREAM_HOLD_TIME 1000000 /*1 ms */
+
+#define MALI_PROFILING_STREAM_BUFFER_SIZE (1 << 12)
+#define MALI_PROFILING_STREAM_BUFFER_NUM 100
+
+/**
+ * Define the mali profiling stream struct.
+ */
+typedef struct mali_profiling_stream {
+ u8 data[MALI_PROFILING_STREAM_BUFFER_SIZE];
+ u32 used_size;
+ struct list_head list;
+} mali_profiling_stream;
+
+typedef struct mali_profiling_stream_list {
+ spinlock_t spin_lock;
+ struct list_head free_list;
+ struct list_head queue_list;
+} mali_profiling_stream_list;
+
+static const char mali_name[] = "4xx";
+static const char utgard_setup_version[] = "ANNOTATE_SETUP 1\n";
+
+static u32 profiling_sample_rate = 0;
+static u32 first_sw_counter_index = 0;
+
+static mali_bool l2_cache_counter_if_enabled = MALI_FALSE;
+static u32 num_counters_enabled = 0;
+static u32 mem_counters_enabled = 0;
+
+static _mali_osk_atomic_t stream_fd_if_used;
+
+static wait_queue_head_t stream_fd_wait_queue;
+static mali_profiling_counter *global_mali_profiling_counters = NULL;
+static u32 num_global_mali_profiling_counters = 0;
+
+static mali_profiling_stream_list *global_mali_stream_list = NULL;
+static mali_profiling_stream *mali_counter_stream = NULL;
+static mali_profiling_stream *mali_core_activity_stream = NULL;
+static u64 mali_core_activity_stream_dequeue_time = 0;
+static spinlock_t mali_activity_lock;
+static u32 mali_activity_cores_num = 0;
+static struct hrtimer profiling_sampling_timer;
+
+const char *_mali_mem_counter_descriptions[] = _MALI_MEM_COUTNER_DESCRIPTIONS;
+const char *_mali_special_counter_descriptions[] = _MALI_SPCIAL_COUNTER_DESCRIPTIONS;
+
+static u32 current_profiling_pid = 0;
+
+static void _mali_profiling_stream_list_destory(mali_profiling_stream_list *profiling_stream_list)
+{
+ mali_profiling_stream *profiling_stream, *tmp_profiling_stream;
+ MALI_DEBUG_ASSERT_POINTER(profiling_stream_list);
+
+ list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->free_list, list) {
+ list_del(&profiling_stream->list);
+ kfree(profiling_stream);
+ }
+
+ list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->queue_list, list) {
+ list_del(&profiling_stream->list);
+ kfree(profiling_stream);
+ }
+
+ kfree(profiling_stream_list);
+}
+
+static void _mali_profiling_global_stream_list_free(void)
+{
+ mali_profiling_stream *profiling_stream, *tmp_profiling_stream;
+ unsigned long irq_flags;
+
+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
+ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags);
+ list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &global_mali_stream_list->queue_list, list) {
+ profiling_stream->used_size = 0;
+ list_move(&profiling_stream->list, &global_mali_stream_list->free_list);
+ }
+ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags);
+}
+
+static _mali_osk_errcode_t _mali_profiling_global_stream_list_dequeue(struct list_head *stream_list, mali_profiling_stream **new_mali_profiling_stream)
+{
+ unsigned long irq_flags;
+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK;
+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
+ MALI_DEBUG_ASSERT_POINTER(stream_list);
+
+ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags);
+
+ if (!list_empty(stream_list)) {
+ *new_mali_profiling_stream = list_entry(stream_list->next, mali_profiling_stream, list);
+ list_del_init(&(*new_mali_profiling_stream)->list);
+ } else {
+ ret = _MALI_OSK_ERR_NOMEM;
+ }
+
+ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags);
+
+ return ret;
+}
+
+static void _mali_profiling_global_stream_list_queue(struct list_head *stream_list, mali_profiling_stream *current_mali_profiling_stream)
+{
+ unsigned long irq_flags;
+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
+ MALI_DEBUG_ASSERT_POINTER(stream_list);
+
+ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags);
+ list_add_tail(&current_mali_profiling_stream->list, stream_list);
+ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags);
+}
+
+static mali_bool _mali_profiling_global_stream_queue_list_if_empty(void)
+{
+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
+ return list_empty(&global_mali_stream_list->queue_list);
+}
+
+static u32 _mali_profiling_global_stream_queue_list_next_size(void)
+{
+ unsigned long irq_flags;
+ u32 size = 0;
+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
+
+ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags);
+ if (!list_empty(&global_mali_stream_list->queue_list)) {
+ mali_profiling_stream *next_mali_profiling_stream =
+ list_entry(global_mali_stream_list->queue_list.next, mali_profiling_stream, list);
+ size = next_mali_profiling_stream->used_size;
+ }
+ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags);
+ return size;
+}
+
+/* The mali profiling stream file operations functions. */
+static ssize_t _mali_profiling_stream_read(
+ struct file *filp,
+ char __user *buffer,
+ size_t size,
+ loff_t *f_pos);
+
+static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait);
+
+static int _mali_profiling_stream_release(struct inode *inode, struct file *filp);
+
+/* The timeline stream file operations structure. */
+static const struct file_operations mali_profiling_stream_fops = {
+ .release = _mali_profiling_stream_release,
+ .read = _mali_profiling_stream_read,
+ .poll = _mali_profiling_stream_poll,
+};
+
+static ssize_t _mali_profiling_stream_read(
+ struct file *filp,
+ char __user *buffer,
+ size_t size,
+ loff_t *f_pos)
+{
+ u32 copy_len = 0;
+ mali_profiling_stream *current_mali_profiling_stream;
+ u32 used_size;
+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
+
+ while (!_mali_profiling_global_stream_queue_list_if_empty()) {
+ used_size = _mali_profiling_global_stream_queue_list_next_size();
+ if (used_size <= ((u32)size - copy_len)) {
+ current_mali_profiling_stream = NULL;
+ _mali_profiling_global_stream_list_dequeue(&global_mali_stream_list->queue_list,
+ &current_mali_profiling_stream);
+ MALI_DEBUG_ASSERT_POINTER(current_mali_profiling_stream);
+ if (copy_to_user(&buffer[copy_len], current_mali_profiling_stream->data, current_mali_profiling_stream->used_size)) {
+ current_mali_profiling_stream->used_size = 0;
+ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream);
+ return -EFAULT;
+ }
+ copy_len += current_mali_profiling_stream->used_size;
+ current_mali_profiling_stream->used_size = 0;
+ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream);
+ } else {
+ break;
+ }
+ }
+ return (ssize_t)copy_len;
+}
+
+static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait)
+{
+ poll_wait(filp, &stream_fd_wait_queue, wait);
+ if (!_mali_profiling_global_stream_queue_list_if_empty())
+ return POLLIN;
+ return 0;
+}
+
+static int _mali_profiling_stream_release(struct inode *inode, struct file *filp)
+{
+ _mali_osk_atomic_init(&stream_fd_if_used, 0);
+ return 0;
+}
+
+/* The funs for control packet and stream data.*/
+static void _mali_profiling_set_packet_size(unsigned char *const buf, const u32 size)
+{
+ u32 i;
+
+ for (i = 0; i < sizeof(size); ++i)
+ buf[i] = (size >> 8 * i) & 0xFF;
+}
+
+static u32 _mali_profiling_get_packet_size(unsigned char *const buf)
+{
+ u32 i;
+ u32 size = 0;
+ for (i = 0; i < sizeof(size); ++i)
+ size |= (u32)buf[i] << 8 * i;
+ return size;
+}
+
+static u32 _mali_profiling_read_packet_int(unsigned char *const buf, u32 *const pos, u32 const packet_size)
+{
+ u64 int_value = 0;
+ u8 shift = 0;
+ u8 byte_value = ~0;
+
+ while ((byte_value & 0x80) != 0) {
+ MALI_DEBUG_ASSERT((*pos) < packet_size);
+ byte_value = buf[*pos];
+ *pos += 1;
+ int_value |= (u32)(byte_value & 0x7f) << shift;
+ shift += 7;
+ }
+
+ if (shift < 8 * sizeof(int_value) && (byte_value & 0x40) != 0) {
+ int_value |= -(1 << shift);
+ }
+
+ return int_value;
+}
+
+static u32 _mali_profiling_pack_int(u8 *const buf, u32 const buf_size, u32 const pos, s32 value)
+{
+ u32 add_bytes = 0;
+ int more = 1;
+ while (more) {
+ /* low order 7 bits of val */
+ char byte_value = value & 0x7f;
+ value >>= 7;
+
+ if ((value == 0 && (byte_value & 0x40) == 0) || (value == -1 && (byte_value & 0x40) != 0)) {
+ more = 0;
+ } else {
+ byte_value |= 0x80;
+ }
+
+ MALI_DEBUG_ASSERT((pos + add_bytes) < buf_size);
+ buf[pos + add_bytes] = byte_value;
+ add_bytes++;
+ }
+
+ return add_bytes;
+}
+
+static int _mali_profiling_pack_long(uint8_t *const buf, u32 const buf_size, u32 const pos, s64 val)
+{
+ int add_bytes = 0;
+ int more = 1;
+ while (more) {
+ /* low order 7 bits of x */
+ char byte_value = val & 0x7f;
+ val >>= 7;
+
+ if ((val == 0 && (byte_value & 0x40) == 0) || (val == -1 && (byte_value & 0x40) != 0)) {
+ more = 0;
+ } else {
+ byte_value |= 0x80;
+ }
+
+ MALI_DEBUG_ASSERT((pos + add_bytes) < buf_size);
+ buf[pos + add_bytes] = byte_value;
+ add_bytes++;
+ }
+
+ return add_bytes;
+}
+
+static void _mali_profiling_stream_add_counter(mali_profiling_stream *profiling_stream, s64 current_time, u32 key, u32 counter_value)
+{
+ u32 add_size = STREAM_HEADER_SIZE;
+ MALI_DEBUG_ASSERT_POINTER(profiling_stream);
+ MALI_DEBUG_ASSERT((profiling_stream->used_size) < MALI_PROFILING_STREAM_BUFFER_SIZE);
+
+ profiling_stream->data[profiling_stream->used_size] = STREAM_HEADER_COUNTER_VALUE;
+
+ add_size += _mali_profiling_pack_long(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE,
+ profiling_stream->used_size + add_size, current_time);
+ add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE,
+ profiling_stream->used_size + add_size, (s32)0);
+ add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE,
+ profiling_stream->used_size + add_size, (s32)key);
+ add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE,
+ profiling_stream->used_size + add_size, (s32)counter_value);
+
+ _mali_profiling_set_packet_size(profiling_stream->data + profiling_stream->used_size + 1,
+ add_size - STREAM_HEADER_SIZE);
+
+ profiling_stream->used_size += add_size;
+}
+
+/* The callback function for sampling timer.*/
+static enum hrtimer_restart _mali_profiling_sampling_counters(struct hrtimer *timer)
+{
+ u32 counter_index;
+ s64 current_time;
+ MALI_DEBUG_ASSERT_POINTER(global_mali_profiling_counters);
+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
+
+ MALI_DEBUG_ASSERT(NULL == mali_counter_stream);
+ if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue(
+ &global_mali_stream_list->free_list, &mali_counter_stream)) {
+
+ MALI_DEBUG_ASSERT_POINTER(mali_counter_stream);
+ MALI_DEBUG_ASSERT(0 == mali_counter_stream->used_size);
+
+ /* Capture l2 cache counter values if enabled */
+ if (MALI_TRUE == l2_cache_counter_if_enabled) {
+ int i, j = 0;
+ _mali_profiling_l2_counter_values l2_counters_values;
+ _mali_profiling_get_l2_counters(&l2_counters_values);
+
+ for (i = COUNTER_L2_0_C0; i <= COUNTER_L2_2_C1; i++) {
+ if (0 == (j % 2))
+ _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value0);
+ else
+ _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value1);
+ j++;
+ }
+ }
+
+ current_time = (s64)_mali_osk_boot_time_get_ns();
+
+ /* Add all enabled counter values into stream */
+ for (counter_index = 0; counter_index < num_global_mali_profiling_counters; counter_index++) {
+ /* No need to sample these couners here. */
+ if (global_mali_profiling_counters[counter_index].enabled) {
+ if ((global_mali_profiling_counters[counter_index].counter_id >= FIRST_MEM_COUNTER &&
+ global_mali_profiling_counters[counter_index].counter_id <= LAST_MEM_COUNTER)
+ || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_VP_ACTIVITY)
+ || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FP_ACTIVITY)
+ || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FILMSTRIP)) {
+
+ continue;
+ }
+
+ if (global_mali_profiling_counters[counter_index].counter_id >= COUNTER_L2_0_C0 &&
+ global_mali_profiling_counters[counter_index].counter_id <= COUNTER_L2_2_C1) {
+
+ u32 prev_val = global_mali_profiling_counters[counter_index].prev_counter_value;
+
+ _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key,
+ global_mali_profiling_counters[counter_index].current_counter_value - prev_val);
+
+ prev_val = global_mali_profiling_counters[counter_index].current_counter_value;
+
+ global_mali_profiling_counters[counter_index].prev_counter_value = prev_val;
+ } else {
+
+ if (global_mali_profiling_counters[counter_index].counter_id == COUNTER_TOTAL_ALLOC_PAGES) {
+ u32 total_alloc_mem = _mali_ukk_report_memory_usage();
+ global_mali_profiling_counters[counter_index].current_counter_value = total_alloc_mem / _MALI_OSK_MALI_PAGE_SIZE;
+ }
+ _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key,
+ global_mali_profiling_counters[counter_index].current_counter_value);
+ if (global_mali_profiling_counters[counter_index].counter_id < FIRST_SPECIAL_COUNTER)
+ global_mali_profiling_counters[counter_index].current_counter_value = 0;
+ }
+ }
+ }
+ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_counter_stream);
+ mali_counter_stream = NULL;
+ } else {
+ MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n"));
+ }
+
+ wake_up_interruptible(&stream_fd_wait_queue);
+
+ /*Enable the sampling timer again*/
+ if (0 != num_counters_enabled && 0 != profiling_sample_rate) {
+ hrtimer_forward_now(&profiling_sampling_timer, ns_to_ktime(profiling_sample_rate));
+ return HRTIMER_RESTART;
+ }
+ return HRTIMER_NORESTART;
+}
+
+static void _mali_profiling_sampling_core_activity_switch(int counter_id, int core, u32 activity, u32 pid)
+{
+ unsigned long irq_flags;
+
+ spin_lock_irqsave(&mali_activity_lock, irq_flags);
+ if (activity == 0)
+ mali_activity_cores_num--;
+ else
+ mali_activity_cores_num++;
+ spin_unlock_irqrestore(&mali_activity_lock, irq_flags);
+
+ if (NULL != global_mali_profiling_counters) {
+ int i ;
+ for (i = 0; i < num_global_mali_profiling_counters; i++) {
+ if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) {
+ u64 current_time = _mali_osk_boot_time_get_ns();
+ u32 add_size = STREAM_HEADER_SIZE;
+
+ if (NULL != mali_core_activity_stream) {
+ if ((mali_core_activity_stream_dequeue_time + MALI_PROFILING_STREAM_HOLD_TIME < current_time) ||
+ (MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE > MALI_PROFILING_STREAM_BUFFER_SIZE
+ - mali_core_activity_stream->used_size)) {
+ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream);
+ mali_core_activity_stream = NULL;
+ wake_up_interruptible(&stream_fd_wait_queue);
+ }
+ }
+
+ if (NULL == mali_core_activity_stream) {
+ if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue(
+ &global_mali_stream_list->free_list, &mali_core_activity_stream)) {
+ mali_core_activity_stream_dequeue_time = current_time;
+ } else {
+ MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n"));
+ wake_up_interruptible(&stream_fd_wait_queue);
+ break;
+ }
+
+ }
+
+ mali_core_activity_stream->data[mali_core_activity_stream->used_size] = STREAM_HEADER_CORE_ACTIVITY;
+
+ add_size += _mali_profiling_pack_long(mali_core_activity_stream->data,
+ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s64)current_time);
+ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data,
+ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, core);
+ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data,
+ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s32)global_mali_profiling_counters[i].key);
+ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data,
+ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, activity);
+ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data,
+ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, pid);
+
+ _mali_profiling_set_packet_size(mali_core_activity_stream->data + mali_core_activity_stream->used_size + 1,
+ add_size - STREAM_HEADER_SIZE);
+
+ mali_core_activity_stream->used_size += add_size;
+
+ if (0 == mali_activity_cores_num) {
+ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream);
+ mali_core_activity_stream = NULL;
+ wake_up_interruptible(&stream_fd_wait_queue);
+ }
+
+ break;
+ }
+ }
+ }
+}
+
+static mali_bool _mali_profiling_global_counters_init(void)
+{
+ int core_id, counter_index, counter_number, counter_id;
+ u32 num_l2_cache_cores;
+ u32 num_pp_cores;
+ u32 num_gp_cores = 1;
+
+ MALI_DEBUG_ASSERT(NULL == global_mali_profiling_counters);
+ num_pp_cores = mali_pp_get_glob_num_pp_cores();
+ num_l2_cache_cores = mali_l2_cache_core_get_glob_num_l2_cores();
+
+ num_global_mali_profiling_counters = 3 * (num_gp_cores + num_pp_cores) + 2 * num_l2_cache_cores
+ + MALI_PROFILING_SW_COUNTERS_NUM
+ + MALI_PROFILING_SPECIAL_COUNTERS_NUM
+ + MALI_PROFILING_MEM_COUNTERS_NUM;
+ global_mali_profiling_counters = _mali_osk_calloc(num_global_mali_profiling_counters, sizeof(mali_profiling_counter));
+
+ if (NULL == global_mali_profiling_counters)
+ return MALI_FALSE;
+
+ counter_index = 0;
+ /*Vertex processor counters */
+ for (core_id = 0; core_id < num_gp_cores; core_id ++) {
+ global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_VP_0 + core_id;
+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_active", mali_name, core_id);
+
+ for (counter_number = 0; counter_number < 2; counter_number++) {
+ counter_index++;
+ global_mali_profiling_counters[counter_index].counter_id = COUNTER_VP_0_C0 + (2 * core_id) + counter_number;
+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_cnt%d", mali_name, core_id, counter_number);
+ }
+ }
+
+ /* Fragment processors' counters */
+ for (core_id = 0; core_id < num_pp_cores; core_id++) {
+ counter_index++;
+ global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_FP_0 + core_id;
+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_active", mali_name, core_id);
+
+ for (counter_number = 0; counter_number < 2; counter_number++) {
+ counter_index++;
+ global_mali_profiling_counters[counter_index].counter_id = COUNTER_FP_0_C0 + (2 * core_id) + counter_number;
+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_cnt%d", mali_name, core_id, counter_number);
+ }
+ }
+
+ /* L2 Cache counters */
+ for (core_id = 0; core_id < num_l2_cache_cores; core_id++) {
+ for (counter_number = 0; counter_number < 2; counter_number++) {
+ counter_index++;
+ global_mali_profiling_counters[counter_index].counter_id = COUNTER_L2_0_C0 + (2 * core_id) + counter_number;
+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_L2_%d_cnt%d", mali_name, core_id, counter_number);
+ }
+ }
+
+ /* Now set up the software counter entries */
+ for (counter_id = FIRST_SW_COUNTER; counter_id <= LAST_SW_COUNTER; counter_id++) {
+ counter_index++;
+
+ if (0 == first_sw_counter_index)
+ first_sw_counter_index = counter_index;
+
+ global_mali_profiling_counters[counter_index].counter_id = counter_id;
+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_SW_%d", mali_name, counter_id - FIRST_SW_COUNTER);
+ }
+
+ /* Now set up the special counter entries */
+ for (counter_id = FIRST_SPECIAL_COUNTER; counter_id <= LAST_SPECIAL_COUNTER; counter_id++) {
+
+ counter_index++;
+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s",
+ mali_name, _mali_special_counter_descriptions[counter_id - FIRST_SPECIAL_COUNTER]);
+
+ global_mali_profiling_counters[counter_index].counter_id = counter_id;
+ }
+
+ /* Now set up the mem counter entries*/
+ for (counter_id = FIRST_MEM_COUNTER; counter_id <= LAST_MEM_COUNTER; counter_id++) {
+
+ counter_index++;
+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s",
+ mali_name, _mali_mem_counter_descriptions[counter_id - FIRST_MEM_COUNTER]);
+
+ global_mali_profiling_counters[counter_index].counter_id = counter_id;
+ }
+
+ MALI_DEBUG_ASSERT((counter_index + 1) == num_global_mali_profiling_counters);
+
+ return MALI_TRUE;
+}
+
+void _mali_profiling_notification_mem_counter(struct mali_session_data *session, u32 counter_id, u32 key, int enable)
+{
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ if (NULL != session) {
+ _mali_osk_notification_t *notification;
+ _mali_osk_notification_queue_t *queue;
+
+ queue = session->ioctl_queue;
+ MALI_DEBUG_ASSERT(NULL != queue);
+
+ notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER,
+ sizeof(_mali_uk_annotate_profiling_mem_counter_s));
+
+ if (NULL != notification) {
+ _mali_uk_annotate_profiling_mem_counter_s *data = notification->result_buffer;
+ data->counter_id = counter_id;
+ data->key = key;
+ data->enable = enable;
+
+ _mali_osk_notification_queue_send(queue, notification);
+ } else {
+ MALI_PRINT_ERROR(("Failed to create notification object!\n"));
+ }
+ } else {
+ MALI_PRINT_ERROR(("Failed to find the right session!\n"));
+ }
+}
+
+void _mali_profiling_notification_enable(struct mali_session_data *session, u32 sampling_rate, int enable)
+{
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ if (NULL != session) {
+ _mali_osk_notification_t *notification;
+ _mali_osk_notification_queue_t *queue;
+
+ queue = session->ioctl_queue;
+ MALI_DEBUG_ASSERT(NULL != queue);
+
+ notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE,
+ sizeof(_mali_uk_annotate_profiling_enable_s));
+
+ if (NULL != notification) {
+ _mali_uk_annotate_profiling_enable_s *data = notification->result_buffer;
+ data->sampling_rate = sampling_rate;
+ data->enable = enable;
+
+ _mali_osk_notification_queue_send(queue, notification);
+ } else {
+ MALI_PRINT_ERROR(("Failed to create notification object!\n"));
+ }
+ } else {
+ MALI_PRINT_ERROR(("Failed to find the right session!\n"));
+ }
+}
+
+
+_mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start)
+{
+ int i;
+ mali_profiling_stream *new_mali_profiling_stream = NULL;
+ mali_profiling_stream_list *new_mali_profiling_stream_list = NULL;
+ if (MALI_TRUE == auto_start) {
+ mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE);
+ }
+
+ /*Init the global_mali_stream_list*/
+ MALI_DEBUG_ASSERT(NULL == global_mali_stream_list);
+ new_mali_profiling_stream_list = (mali_profiling_stream_list *)kmalloc(sizeof(mali_profiling_stream_list), GFP_KERNEL);
+
+ if (NULL == new_mali_profiling_stream_list) {
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ spin_lock_init(&new_mali_profiling_stream_list->spin_lock);
+ INIT_LIST_HEAD(&new_mali_profiling_stream_list->free_list);
+ INIT_LIST_HEAD(&new_mali_profiling_stream_list->queue_list);
+
+ spin_lock_init(&mali_activity_lock);
+ mali_activity_cores_num = 0;
+
+ for (i = 0; i < MALI_PROFILING_STREAM_BUFFER_NUM; i++) {
+ new_mali_profiling_stream = (mali_profiling_stream *)kmalloc(sizeof(mali_profiling_stream), GFP_KERNEL);
+ if (NULL == new_mali_profiling_stream) {
+ _mali_profiling_stream_list_destory(new_mali_profiling_stream_list);
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ INIT_LIST_HEAD(&new_mali_profiling_stream->list);
+ new_mali_profiling_stream->used_size = 0;
+ list_add_tail(&new_mali_profiling_stream->list, &new_mali_profiling_stream_list->free_list);
+
+ }
+
+ _mali_osk_atomic_init(&stream_fd_if_used, 0);
+ init_waitqueue_head(&stream_fd_wait_queue);
+
+ hrtimer_init(&profiling_sampling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+
+ profiling_sampling_timer.function = _mali_profiling_sampling_counters;
+
+ global_mali_stream_list = new_mali_profiling_stream_list;
+
+ return _MALI_OSK_ERR_OK;
+}
+
+void _mali_osk_profiling_term(void)
+{
+ if (0 != profiling_sample_rate) {
+ hrtimer_cancel(&profiling_sampling_timer);
+ profiling_sample_rate = 0;
+ }
+ _mali_osk_atomic_term(&stream_fd_if_used);
+
+ if (NULL != global_mali_profiling_counters) {
+ _mali_osk_free(global_mali_profiling_counters);
+ global_mali_profiling_counters = NULL;
+ num_global_mali_profiling_counters = 0;
+ }
+
+ if (NULL != global_mali_stream_list) {
+ _mali_profiling_stream_list_destory(global_mali_stream_list);
+ global_mali_stream_list = NULL;
+ }
+
+}
+
+void _mali_osk_profiling_stop_sampling(u32 pid)
+{
+ if (pid == current_profiling_pid) {
+
+ int i;
+ /* Reset all counter states when closing connection.*/
+ for (i = 0; i < num_global_mali_profiling_counters; ++i) {
+ _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER);
+ global_mali_profiling_counters[i].enabled = 0;
+ global_mali_profiling_counters[i].prev_counter_value = 0;
+ global_mali_profiling_counters[i].current_counter_value = 0;
+ }
+ l2_cache_counter_if_enabled = MALI_FALSE;
+ num_counters_enabled = 0;
+ mem_counters_enabled = 0;
+ _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0);
+ _mali_profiling_control(SW_COUNTER_ENABLE, 0);
+ /* Delete sampling timer when closing connection. */
+ if (0 != profiling_sample_rate) {
+ hrtimer_cancel(&profiling_sampling_timer);
+ profiling_sample_rate = 0;
+ }
+ current_profiling_pid = 0;
+ }
+}
+
+void _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4)
+{
+ /*Record the freq & volt to global_mali_profiling_counters here. */
+ if (0 != profiling_sample_rate) {
+ u32 channel;
+ u32 state;
+ channel = (event_id >> 16) & 0xFF;
+ state = ((event_id >> 24) & 0xF) << 24;
+
+ switch (state) {
+ case MALI_PROFILING_EVENT_TYPE_SINGLE:
+ if ((MALI_PROFILING_EVENT_CHANNEL_GPU >> 16) == channel) {
+ u32 reason = (event_id & 0xFFFF);
+ if (MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE == reason) {
+ _mali_osk_profiling_record_global_counters(COUNTER_FREQUENCY, data0);
+ _mali_osk_profiling_record_global_counters(COUNTER_VOLTAGE, data1);
+ }
+ }
+ break;
+ case MALI_PROFILING_EVENT_TYPE_START:
+ if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) {
+ _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 1, data1);
+ } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) &&
+ (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) {
+ u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16);
+ _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 1, data1);
+ }
+ break;
+ case MALI_PROFILING_EVENT_TYPE_STOP:
+ if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) {
+ _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 0, 0);
+ } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) &&
+ (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) {
+ u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16);
+ _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 0, 0);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ trace_mali_timeline_event(event_id, data0, data1, data2, data3, data4);
+}
+
+void _mali_osk_profiling_report_sw_counters(u32 *counters)
+{
+ trace_mali_sw_counters(_mali_osk_get_pid(), _mali_osk_get_tid(), NULL, counters);
+}
+
+void _mali_osk_profiling_record_global_counters(int counter_id, u32 value)
+{
+ if (NULL != global_mali_profiling_counters) {
+ int i ;
+ for (i = 0; i < num_global_mali_profiling_counters; i++) {
+ if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) {
+ global_mali_profiling_counters[i].current_counter_value = value;
+ break;
+ }
+ }
+ }
+}
+
+_mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args)
+{
+ /* Always add process and thread identificator in the first two data elements for events from user space */
+ _mali_osk_profiling_add_event(args->event_id, _mali_osk_get_pid(), _mali_osk_get_tid(), args->data[2], args->data[3], args->data[4]);
+
+ return _MALI_OSK_ERR_OK;
+}
+
+_mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s *args)
+{
+ u32 *counters = (u32 *)(uintptr_t)args->counters;
+
+ _mali_osk_profiling_report_sw_counters(counters);
+
+ if (NULL != global_mali_profiling_counters) {
+ int i;
+ for (i = 0; i < MALI_PROFILING_SW_COUNTERS_NUM; i ++) {
+ if (global_mali_profiling_counters[first_sw_counter_index + i].enabled) {
+ global_mali_profiling_counters[first_sw_counter_index + i].current_counter_value = *(counters + i);
+ }
+ }
+ }
+
+ return _MALI_OSK_ERR_OK;
+}
+
+_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args)
+{
+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ if (1 == _mali_osk_atomic_inc_return(&stream_fd_if_used)) {
+
+ s32 fd = anon_inode_getfd("[mali_profiling_stream]", &mali_profiling_stream_fops,
+ session,
+ O_RDONLY | O_CLOEXEC);
+
+ args->stream_fd = fd;
+ if (0 > fd) {
+ _mali_osk_atomic_dec(&stream_fd_if_used);
+ return _MALI_OSK_ERR_FAULT;
+ }
+ args->stream_fd = fd;
+ } else {
+ _mali_osk_atomic_dec(&stream_fd_if_used);
+ args->stream_fd = -1;
+ return _MALI_OSK_ERR_BUSY;
+ }
+
+ return _MALI_OSK_ERR_OK;
+}
+
+_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args)
+{
+ u32 control_packet_size;
+ u32 output_buffer_size;
+
+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ if (NULL == global_mali_profiling_counters && MALI_FALSE == _mali_profiling_global_counters_init()) {
+ MALI_PRINT_ERROR(("Failed to create global_mali_profiling_counters.\n"));
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ control_packet_size = args->control_packet_size;
+ output_buffer_size = args->response_packet_size;
+
+ if (0 != control_packet_size) {
+ u8 control_type;
+ u8 *control_packet_data;
+ u8 *response_packet_data;
+ u32 version_length = sizeof(utgard_setup_version) - 1;
+
+ control_packet_data = (u8 *)(uintptr_t)args->control_packet_data;
+ MALI_DEBUG_ASSERT_POINTER(control_packet_data);
+ response_packet_data = (u8 *)(uintptr_t)args->response_packet_data;
+ MALI_DEBUG_ASSERT_POINTER(response_packet_data);
+
+ /*Decide if need to ignore Utgard setup version.*/
+ if (control_packet_size >= version_length) {
+ if (0 == memcmp(control_packet_data, utgard_setup_version, version_length)) {
+ if (control_packet_size == version_length) {
+ args->response_packet_size = 0;
+ return _MALI_OSK_ERR_OK;
+ } else {
+ control_packet_data += version_length;
+ control_packet_size -= version_length;
+ }
+ }
+ }
+
+ current_profiling_pid = _mali_osk_get_pid();
+
+ control_type = control_packet_data[0];
+ switch (control_type) {
+ case PACKET_HEADER_COUNTERS_REQUEST: {
+ int i;
+
+ if (PACKET_HEADER_SIZE > control_packet_size ||
+ control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) {
+ MALI_PRINT_ERROR(("Wrong control packet size, type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size));
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ /* Send supported counters */
+ *response_packet_data = PACKET_HEADER_COUNTERS_ACK;
+ args->response_packet_size = PACKET_HEADER_SIZE;
+
+ for (i = 0; i < num_global_mali_profiling_counters; ++i) {
+ u32 name_size = strlen(global_mali_profiling_counters[i].counter_name);
+
+ if ((args->response_packet_size + name_size + 1) > output_buffer_size) {
+ MALI_PRINT_ERROR(("Response packet data is too large..\n"));
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ memcpy(response_packet_data + args->response_packet_size,
+ global_mali_profiling_counters[i].counter_name, name_size + 1);
+
+ args->response_packet_size += (name_size + 1);
+
+ if (global_mali_profiling_counters[i].counter_id == COUNTER_VP_ACTIVITY) {
+ args->response_packet_size += _mali_profiling_pack_int(response_packet_data,
+ output_buffer_size, args->response_packet_size, (s32)1);
+ } else if (global_mali_profiling_counters[i].counter_id == COUNTER_FP_ACTIVITY) {
+ args->response_packet_size += _mali_profiling_pack_int(response_packet_data,
+ output_buffer_size, args->response_packet_size, (s32)mali_pp_get_glob_num_pp_cores());
+ } else {
+ args->response_packet_size += _mali_profiling_pack_int(response_packet_data,
+ output_buffer_size, args->response_packet_size, (s32) - 1);
+ }
+ }
+
+ _mali_profiling_set_packet_size(response_packet_data + 1, args->response_packet_size);
+ break;
+ }
+
+ case PACKET_HEADER_COUNTERS_ENABLE: {
+ int i;
+ u32 request_pos = PACKET_HEADER_SIZE;
+ mali_bool sw_counter_if_enabled = MALI_FALSE;
+
+ if (PACKET_HEADER_SIZE > control_packet_size ||
+ control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) {
+ MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size));
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ /* Init all counter states before enable requested counters.*/
+ for (i = 0; i < num_global_mali_profiling_counters; ++i) {
+ _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER);
+ global_mali_profiling_counters[i].enabled = 0;
+ global_mali_profiling_counters[i].prev_counter_value = 0;
+ global_mali_profiling_counters[i].current_counter_value = 0;
+
+ if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER &&
+ global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) {
+ _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, 0, 0);
+ }
+ }
+
+ l2_cache_counter_if_enabled = MALI_FALSE;
+ num_counters_enabled = 0;
+ mem_counters_enabled = 0;
+ _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0);
+ _mali_profiling_control(SW_COUNTER_ENABLE, 0);
+ _mali_profiling_notification_enable(session, 0, 0);
+
+ /* Enable requested counters */
+ while (request_pos < control_packet_size) {
+ u32 begin = request_pos;
+ u32 event;
+ u32 key;
+
+ while (request_pos < control_packet_size && control_packet_data[request_pos] != '\0') {
+ ++request_pos;
+ }
+
+ ++request_pos;
+ event = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size);
+ key = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size);
+
+ for (i = 0; i < num_global_mali_profiling_counters; ++i) {
+ u32 name_size = strlen((char *)(control_packet_data + begin));
+ if (strncmp(global_mali_profiling_counters[i].counter_name, (char *)(control_packet_data + begin), name_size) == 0) {
+ if (!sw_counter_if_enabled && (FIRST_SW_COUNTER <= global_mali_profiling_counters[i].counter_id
+ && global_mali_profiling_counters[i].counter_id <= LAST_SW_COUNTER)) {
+ sw_counter_if_enabled = MALI_TRUE;
+ _mali_profiling_control(SW_COUNTER_ENABLE, 1);
+ }
+
+ if (COUNTER_FILMSTRIP == global_mali_profiling_counters[i].counter_id) {
+ _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 1);
+ _mali_profiling_control(FBDUMP_CONTROL_RATE, event & 0xff);
+ _mali_profiling_control(FBDUMP_CONTROL_RESIZE_FACTOR, (event >> 8) & 0xff);
+ }
+
+ if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER &&
+ global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) {
+ _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id,
+ key, 1);
+ mem_counters_enabled++;
+ }
+
+ global_mali_profiling_counters[i].counter_event = event;
+ global_mali_profiling_counters[i].key = key;
+ global_mali_profiling_counters[i].enabled = 1;
+
+ _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id,
+ global_mali_profiling_counters[i].counter_event);
+ num_counters_enabled++;
+ break;
+ }
+ }
+
+ if (i == num_global_mali_profiling_counters) {
+ MALI_PRINT_ERROR(("Counter name does not match for type %u.\n", control_type));
+ return _MALI_OSK_ERR_FAULT;
+ }
+ }
+
+ if (PACKET_HEADER_SIZE <= output_buffer_size) {
+ *response_packet_data = PACKET_HEADER_ACK;
+ _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE);
+ args->response_packet_size = PACKET_HEADER_SIZE;
+ } else {
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ break;
+ }
+
+ case PACKET_HEADER_START_CAPTURE_VALUE: {
+ u32 live_rate;
+ u32 request_pos = PACKET_HEADER_SIZE;
+
+ if (PACKET_HEADER_SIZE > control_packet_size ||
+ control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) {
+ MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size));
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ /* Read samping rate in nanoseconds and live rate, start capture.*/
+ profiling_sample_rate = _mali_profiling_read_packet_int(control_packet_data,
+ &request_pos, control_packet_size);
+
+ live_rate = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size);
+
+ if (PACKET_HEADER_SIZE <= output_buffer_size) {
+ *response_packet_data = PACKET_HEADER_ACK;
+ _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE);
+ args->response_packet_size = PACKET_HEADER_SIZE;
+ } else {
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ if (0 != num_counters_enabled && 0 != profiling_sample_rate) {
+ _mali_profiling_global_stream_list_free();
+ if (mem_counters_enabled > 0) {
+ _mali_profiling_notification_enable(session, profiling_sample_rate, 1);
+ }
+ hrtimer_start(&profiling_sampling_timer,
+ ktime_set(profiling_sample_rate / 1000000000, profiling_sample_rate % 1000000000),
+ HRTIMER_MODE_REL_PINNED);
+ }
+
+ break;
+ }
+ default:
+ MALI_PRINT_ERROR(("Unsupported profiling packet header type %u.\n", control_type));
+ args->response_packet_size = 0;
+ return _MALI_OSK_ERR_FAULT;
+ }
+ } else {
+ _mali_osk_profiling_stop_sampling(current_profiling_pid);
+ _mali_profiling_notification_enable(session, 0, 0);
+ }
+
+ return _MALI_OSK_ERR_OK;
+}
+
+/**
+ * Called by gator.ko to set HW counters
+ *
+ * @param counter_id The counter ID.
+ * @param event_id Event ID that the counter should count (HW counter value from TRM).
+ *
+ * @return 1 on success, 0 on failure.
+ */
+int _mali_profiling_set_event(u32 counter_id, s32 event_id)
+{
+ if (COUNTER_VP_0_C0 == counter_id) {
+ mali_gp_job_set_gp_counter_src0(event_id);
+ } else if (COUNTER_VP_0_C1 == counter_id) {
+ mali_gp_job_set_gp_counter_src1(event_id);
+ } else if (COUNTER_FP_0_C0 <= counter_id && COUNTER_FP_7_C1 >= counter_id) {
+ /*
+ * Two compatibility notes for this function:
+ *
+ * 1) Previously the DDK allowed per core counters.
+ *
+ * This did not make much sense on Mali-450 with the "virtual PP core" concept,
+ * so this option was removed, and only the same pair of HW counters was allowed on all cores,
+ * beginning with r3p2 release.
+ *
+ * Starting with r4p0, it is now possible to set different HW counters for the different sub jobs.
+ * This should be almost the same, since sub job 0 is designed to run on core 0,
+ * sub job 1 on core 1, and so on.
+ *
+ * The scheduling of PP sub jobs is not predictable, and this often led to situations where core 0 ran 2
+ * sub jobs, while for instance core 1 ran zero. Having the counters set per sub job would thus increase
+ * the predictability of the returned data (as you would be guaranteed data for all the selected HW counters).
+ *
+ * PS: Core scaling needs to be disabled in order to use this reliably (goes for both solutions).
+ *
+ * The framework/#defines with Gator still indicates that the counter is for a particular core,
+ * but this is internally used as a sub job ID instead (no translation needed).
+ *
+ * 2) Global/default vs per sub job counters
+ *
+ * Releases before r3p2 had only per PP core counters.
+ * r3p2 releases had only one set of default/global counters which applied to all PP cores
+ * Starting with r4p0, we have both a set of default/global counters,
+ * and individual counters per sub job (equal to per core).
+ *
+ * To keep compatibility with Gator/DS-5/streamline, the following scheme is used:
+ *
+ * r3p2 release; only counters set for core 0 is handled,
+ * this is applied as the default/global set of counters, and will thus affect all cores.
+ *
+ * r4p0 release; counters set for core 0 is applied as both the global/default set of counters,
+ * and counters for sub job 0.
+ * Counters set for core 1-7 is only applied for the corresponding sub job.
+ *
+ * This should allow the DS-5/Streamline GUI to have a simple mode where it only allows setting the
+ * values for core 0, and thus this will be applied to all PP sub jobs/cores.
+ * Advanced mode will also be supported, where individual pairs of HW counters can be selected.
+ *
+ * The GUI will (until it is updated) still refer to cores instead of sub jobs, but this is probably
+ * something we can live with!
+ *
+ * Mali-450 note: Each job is not divided into a deterministic number of sub jobs, as the HW DLBU
+ * automatically distributes the load between whatever number of cores is available at this particular time.
+ * A normal PP job on Mali-450 is thus considered a single (virtual) job, and it will thus only be possible
+ * to use a single pair of HW counters (even if the job ran on multiple PP cores).
+ * In other words, only the global/default pair of PP HW counters will be used for normal Mali-450 jobs.
+ */
+ u32 sub_job = (counter_id - COUNTER_FP_0_C0) >> 1;
+ u32 counter_src = (counter_id - COUNTER_FP_0_C0) & 1;
+ if (0 == counter_src) {
+ mali_pp_job_set_pp_counter_sub_job_src0(sub_job, event_id);
+ if (0 == sub_job) {
+ mali_pp_job_set_pp_counter_global_src0(event_id);
+ }
+ } else {
+ mali_pp_job_set_pp_counter_sub_job_src1(sub_job, event_id);
+ if (0 == sub_job) {
+ mali_pp_job_set_pp_counter_global_src1(event_id);
+ }
+ }
+ } else if (COUNTER_L2_0_C0 <= counter_id && COUNTER_L2_2_C1 >= counter_id) {
+ u32 core_id = (counter_id - COUNTER_L2_0_C0) >> 1;
+ struct mali_l2_cache_core *l2_cache_core = mali_l2_cache_core_get_glob_l2_core(core_id);
+
+ if (NULL != l2_cache_core) {
+ u32 counter_src = (counter_id - COUNTER_L2_0_C0) & 1;
+ mali_l2_cache_core_set_counter_src(l2_cache_core,
+ counter_src, event_id);
+ l2_cache_counter_if_enabled = MALI_TRUE;
+ }
+ } else {
+ return 0; /* Failure, unknown event */
+ }
+
+ return 1; /* success */
+}
+
+/**
+ * Called by gator.ko to retrieve the L2 cache counter values for all L2 cache cores.
+ * The L2 cache counters are unique in that they are polled by gator, rather than being
+ * transmitted via the tracepoint mechanism.
+ *
+ * @param values Pointer to a _mali_profiling_l2_counter_values structure where
+ * the counter sources and values will be output
+ * @return 0 if all went well; otherwise, return the mask with the bits set for the powered off cores
+ */
+u32 _mali_profiling_get_l2_counters(_mali_profiling_l2_counter_values *values)
+{
+ u32 l2_cores_num = mali_l2_cache_core_get_glob_num_l2_cores();
+ u32 i;
+
+ MALI_DEBUG_ASSERT(l2_cores_num <= 3);
+
+ for (i = 0; i < l2_cores_num; i++) {
+ struct mali_l2_cache_core *l2_cache = mali_l2_cache_core_get_glob_l2_core(i);
+
+ if (NULL == l2_cache) {
+ continue;
+ }
+
+ mali_l2_cache_core_get_counter_values(l2_cache,
+ &values->cores[i].source0,
+ &values->cores[i].value0,
+ &values->cores[i].source1,
+ &values->cores[i].value1);
+ }
+
+ return 0;
+}
+
+/**
+ * Called by gator to control the production of profiling information at runtime.
+ */
+void _mali_profiling_control(u32 action, u32 value)
+{
+ switch (action) {
+ case FBDUMP_CONTROL_ENABLE:
+ mali_set_user_setting(_MALI_UK_USER_SETTING_COLORBUFFER_CAPTURE_ENABLED, (value == 0 ? MALI_FALSE : MALI_TRUE));
+ break;
+ case FBDUMP_CONTROL_RATE:
+ mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_N_FRAMES, value);
+ break;
+ case SW_COUNTER_ENABLE:
+ mali_set_user_setting(_MALI_UK_USER_SETTING_SW_COUNTER_ENABLED, value);
+ break;
+ case FBDUMP_CONTROL_RESIZE_FACTOR:
+ mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_RESIZE_FACTOR, value);
+ break;
+ default:
+ break; /* Ignore unimplemented actions */
+ }
+}
+
+/**
+ * Called by gator to get mali api version.
+ */
+u32 _mali_profiling_get_api_version(void)
+{
+ return MALI_PROFILING_API_VERSION;
+}
+
+/**
+* Called by gator to get the data about Mali instance in use:
+* product id, version, number of cores
+*/
+void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values)
+{
+ values->mali_product_id = (u32)mali_kernel_core_get_product_id();
+ values->mali_version_major = mali_kernel_core_get_gpu_major_version();
+ values->mali_version_minor = mali_kernel_core_get_gpu_minor_version();
+ values->num_of_l2_cores = mali_l2_cache_core_get_glob_num_l2_cores();
+ values->num_of_fp_cores = mali_executor_get_num_cores_total();
+ values->num_of_vp_cores = 1;
+}
+
+
+EXPORT_SYMBOL(_mali_profiling_set_event);
+EXPORT_SYMBOL(_mali_profiling_get_l2_counters);
+EXPORT_SYMBOL(_mali_profiling_control);
+EXPORT_SYMBOL(_mali_profiling_get_api_version);
+EXPORT_SYMBOL(_mali_profiling_get_mali_version);
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_specific.h b/drivers/gpu/arm/utgard/linux/mali_osk_specific.h
new file mode 100644
index 000000000000..db034a5b3c70
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_specific.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_specific.h
+ * Defines per-OS Kernel level specifics, such as unusual workarounds for
+ * certain OSs.
+ */
+
+#ifndef __MALI_OSK_SPECIFIC_H__
+#define __MALI_OSK_SPECIFIC_H__
+
+#include <asm/uaccess.h>
+#include <linux/platform_device.h>
+#include <linux/gfp.h>
+#include <linux/hardirq.h>
+
+
+#include "mali_osk_types.h"
+#include "mali_kernel_linux.h"
+
+#define MALI_STATIC_INLINE static inline
+#define MALI_NON_STATIC_INLINE inline
+
+typedef struct dma_pool *mali_dma_pool;
+
+typedef u32 mali_dma_addr;
+
+#if MALI_ENABLE_CPU_CYCLES
+/* Reads out the clock cycle performance counter of the current cpu.
+ It is useful for cost-free (2 cycle) measuring of the time spent
+ in a code path. Sample before and after, the diff number of cycles.
+ When the CPU is idle it will not increase this clock counter.
+ It means that the counter is accurate if only spin-locks are used,
+ but mutexes may lead to too low values since the cpu might "idle"
+ waiting for the mutex to become available.
+ The clock source is configured on the CPU during mali module load,
+ but will not give useful output after a CPU has been power cycled.
+ It is therefore important to configure the system to not turn of
+ the cpu cores when using this functionallity.*/
+static inline unsigned int mali_get_cpu_cyclecount(void)
+{
+ unsigned int value;
+ /* Reading the CCNT Register - CPU clock counter */
+ asm volatile("MRC p15, 0, %0, c9, c13, 0\t\n": "=r"(value));
+ return value;
+}
+
+void mali_init_cpu_time_counters(int reset, int enable_divide_by_64);
+#endif
+
+
+MALI_STATIC_INLINE u32 _mali_osk_copy_from_user(void *to, void *from, u32 n)
+{
+ return (u32)copy_from_user(to, from, (unsigned long)n);
+}
+
+MALI_STATIC_INLINE mali_bool _mali_osk_in_atomic(void)
+{
+ return in_atomic();
+}
+
+#define _mali_osk_put_user(x, ptr) put_user(x, ptr)
+
+#endif /* __MALI_OSK_SPECIFIC_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_time.c b/drivers/gpu/arm/utgard/linux/mali_osk_time.c
new file mode 100644
index 000000000000..4deaa101e48f
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_time.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_time.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+
+#include "mali_osk.h"
+#include <linux/jiffies.h>
+#include <linux/time.h>
+#include <asm/delay.h>
+
+mali_bool _mali_osk_time_after_eq(unsigned long ticka, unsigned long tickb)
+{
+ return time_after_eq(ticka, tickb) ?
+ MALI_TRUE : MALI_FALSE;
+}
+
+unsigned long _mali_osk_time_mstoticks(u32 ms)
+{
+ return msecs_to_jiffies(ms);
+}
+
+u32 _mali_osk_time_tickstoms(unsigned long ticks)
+{
+ return jiffies_to_msecs(ticks);
+}
+
+unsigned long _mali_osk_time_tickcount(void)
+{
+ return jiffies;
+}
+
+void _mali_osk_time_ubusydelay(u32 usecs)
+{
+ udelay(usecs);
+}
+
+u64 _mali_osk_time_get_ns(void)
+{
+ struct timespec tsval;
+ getnstimeofday(&tsval);
+ return (u64)timespec_to_ns(&tsval);
+}
+
+u64 _mali_osk_boot_time_get_ns(void)
+{
+ struct timespec tsval;
+ get_monotonic_boottime(&tsval);
+ return (u64)timespec_to_ns(&tsval);
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_timers.c b/drivers/gpu/arm/utgard/linux/mali_osk_timers.c
new file mode 100644
index 000000000000..6bbaee749d64
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_timers.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_timers.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+
+#include <linux/timer.h>
+#include <linux/slab.h>
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+
+struct _mali_osk_timer_t_struct {
+ struct timer_list timer;
+};
+
+typedef void (*timer_timeout_function_t)(unsigned long);
+
+_mali_osk_timer_t *_mali_osk_timer_init(void)
+{
+ _mali_osk_timer_t *t = (_mali_osk_timer_t *)kmalloc(sizeof(_mali_osk_timer_t), GFP_KERNEL);
+ if (NULL != t) init_timer(&t->timer);
+ return t;
+}
+
+void _mali_osk_timer_add(_mali_osk_timer_t *tim, unsigned long ticks_to_expire)
+{
+ MALI_DEBUG_ASSERT_POINTER(tim);
+ tim->timer.expires = jiffies + ticks_to_expire;
+ add_timer(&(tim->timer));
+}
+
+void _mali_osk_timer_mod(_mali_osk_timer_t *tim, unsigned long ticks_to_expire)
+{
+ MALI_DEBUG_ASSERT_POINTER(tim);
+ mod_timer(&(tim->timer), jiffies + ticks_to_expire);
+}
+
+void _mali_osk_timer_del(_mali_osk_timer_t *tim)
+{
+ MALI_DEBUG_ASSERT_POINTER(tim);
+ del_timer_sync(&(tim->timer));
+}
+
+void _mali_osk_timer_del_async(_mali_osk_timer_t *tim)
+{
+ MALI_DEBUG_ASSERT_POINTER(tim);
+ del_timer(&(tim->timer));
+}
+
+mali_bool _mali_osk_timer_pending(_mali_osk_timer_t *tim)
+{
+ MALI_DEBUG_ASSERT_POINTER(tim);
+ return 1 == timer_pending(&(tim->timer));
+}
+
+void _mali_osk_timer_setcallback(_mali_osk_timer_t *tim, _mali_osk_timer_callback_t callback, void *data)
+{
+ MALI_DEBUG_ASSERT_POINTER(tim);
+ tim->timer.data = (unsigned long)data;
+ tim->timer.function = (timer_timeout_function_t)callback;
+}
+
+void _mali_osk_timer_term(_mali_osk_timer_t *tim)
+{
+ MALI_DEBUG_ASSERT_POINTER(tim);
+ kfree(tim);
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_wait_queue.c b/drivers/gpu/arm/utgard/linux/mali_osk_wait_queue.c
new file mode 100644
index 000000000000..15d5ce250eb1
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_wait_queue.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_wait_queue.c
+ * Implemenation of the OS abstraction layer for the kernel device driver
+ */
+
+#include <linux/wait.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+
+struct _mali_osk_wait_queue_t_struct {
+ wait_queue_head_t wait_queue;
+};
+
+_mali_osk_wait_queue_t *_mali_osk_wait_queue_init(void)
+{
+ _mali_osk_wait_queue_t *ret = NULL;
+
+ ret = kmalloc(sizeof(_mali_osk_wait_queue_t), GFP_KERNEL);
+
+ if (NULL == ret) {
+ return ret;
+ }
+
+ init_waitqueue_head(&ret->wait_queue);
+ MALI_DEBUG_ASSERT(!waitqueue_active(&ret->wait_queue));
+
+ return ret;
+}
+
+void _mali_osk_wait_queue_wait_event(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data)
+{
+ MALI_DEBUG_ASSERT_POINTER(queue);
+ MALI_DEBUG_PRINT(6, ("Adding to wait queue %p\n", queue));
+ wait_event(queue->wait_queue, condition(data));
+}
+
+void _mali_osk_wait_queue_wait_event_timeout(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data, u32 timeout)
+{
+ MALI_DEBUG_ASSERT_POINTER(queue);
+ MALI_DEBUG_PRINT(6, ("Adding to wait queue %p\n", queue));
+ wait_event_timeout(queue->wait_queue, condition(data), _mali_osk_time_mstoticks(timeout));
+}
+
+void _mali_osk_wait_queue_wake_up(_mali_osk_wait_queue_t *queue)
+{
+ MALI_DEBUG_ASSERT_POINTER(queue);
+
+ /* if queue is empty, don't attempt to wake up its elements */
+ if (!waitqueue_active(&queue->wait_queue)) return;
+
+ MALI_DEBUG_PRINT(6, ("Waking up elements in wait queue %p ....\n", queue));
+
+ wake_up_all(&queue->wait_queue);
+
+ MALI_DEBUG_PRINT(6, ("... elements in wait queue %p woken up\n", queue));
+}
+
+void _mali_osk_wait_queue_term(_mali_osk_wait_queue_t *queue)
+{
+ /* Parameter validation */
+ MALI_DEBUG_ASSERT_POINTER(queue);
+
+ /* Linux requires no explicit termination of wait queues */
+ kfree(queue);
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_wq.c b/drivers/gpu/arm/utgard/linux/mali_osk_wq.c
new file mode 100644
index 000000000000..2c34c91a7922
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_osk_wq.c
@@ -0,0 +1,240 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_osk_wq.c
+ * Implementation of the OS abstraction layer for the kernel device driver
+ */
+
+#include <linux/slab.h> /* For memory allocation */
+#include <linux/workqueue.h>
+#include <linux/version.h>
+#include <linux/sched.h>
+
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_kernel_license.h"
+#include "mali_kernel_linux.h"
+
+typedef struct _mali_osk_wq_work_s {
+ _mali_osk_wq_work_handler_t handler;
+ void *data;
+ mali_bool high_pri;
+ struct work_struct work_handle;
+} mali_osk_wq_work_object_t;
+
+typedef struct _mali_osk_wq_delayed_work_s {
+ _mali_osk_wq_work_handler_t handler;
+ void *data;
+ struct delayed_work work;
+} mali_osk_wq_delayed_work_object_t;
+
+#if MALI_LICENSE_IS_GPL
+static struct workqueue_struct *mali_wq_normal = NULL;
+static struct workqueue_struct *mali_wq_high = NULL;
+#endif
+
+static void _mali_osk_wq_work_func(struct work_struct *work);
+
+_mali_osk_errcode_t _mali_osk_wq_init(void)
+{
+#if MALI_LICENSE_IS_GPL
+ MALI_DEBUG_ASSERT(NULL == mali_wq_normal);
+ MALI_DEBUG_ASSERT(NULL == mali_wq_high);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)
+ mali_wq_normal = alloc_workqueue("mali", WQ_UNBOUND, 0);
+ mali_wq_high = alloc_workqueue("mali_high_pri", WQ_HIGHPRI | WQ_UNBOUND, 0);
+#else
+ mali_wq_normal = create_workqueue("mali");
+ mali_wq_high = create_workqueue("mali_high_pri");
+#endif
+ if (NULL == mali_wq_normal || NULL == mali_wq_high) {
+ MALI_PRINT_ERROR(("Unable to create Mali workqueues\n"));
+
+ if (mali_wq_normal) destroy_workqueue(mali_wq_normal);
+ if (mali_wq_high) destroy_workqueue(mali_wq_high);
+
+ mali_wq_normal = NULL;
+ mali_wq_high = NULL;
+
+ return _MALI_OSK_ERR_FAULT;
+ }
+#endif /* MALI_LICENSE_IS_GPL */
+
+ return _MALI_OSK_ERR_OK;
+}
+
+void _mali_osk_wq_flush(void)
+{
+#if MALI_LICENSE_IS_GPL
+ flush_workqueue(mali_wq_high);
+ flush_workqueue(mali_wq_normal);
+#else
+ flush_scheduled_work();
+#endif
+}
+
+void _mali_osk_wq_term(void)
+{
+#if MALI_LICENSE_IS_GPL
+ MALI_DEBUG_ASSERT(NULL != mali_wq_normal);
+ MALI_DEBUG_ASSERT(NULL != mali_wq_high);
+
+ flush_workqueue(mali_wq_normal);
+ destroy_workqueue(mali_wq_normal);
+
+ flush_workqueue(mali_wq_high);
+ destroy_workqueue(mali_wq_high);
+
+ mali_wq_normal = NULL;
+ mali_wq_high = NULL;
+#else
+ flush_scheduled_work();
+#endif
+}
+
+_mali_osk_wq_work_t *_mali_osk_wq_create_work(_mali_osk_wq_work_handler_t handler, void *data)
+{
+ mali_osk_wq_work_object_t *work = kmalloc(sizeof(mali_osk_wq_work_object_t), GFP_KERNEL);
+
+ if (NULL == work) return NULL;
+
+ work->handler = handler;
+ work->data = data;
+ work->high_pri = MALI_FALSE;
+
+ INIT_WORK(&work->work_handle, _mali_osk_wq_work_func);
+
+ return work;
+}
+
+_mali_osk_wq_work_t *_mali_osk_wq_create_work_high_pri(_mali_osk_wq_work_handler_t handler, void *data)
+{
+ mali_osk_wq_work_object_t *work = kmalloc(sizeof(mali_osk_wq_work_object_t), GFP_KERNEL);
+
+ if (NULL == work) return NULL;
+
+ work->handler = handler;
+ work->data = data;
+ work->high_pri = MALI_TRUE;
+
+ INIT_WORK(&work->work_handle, _mali_osk_wq_work_func);
+
+ return work;
+}
+
+void _mali_osk_wq_delete_work(_mali_osk_wq_work_t *work)
+{
+ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work;
+ _mali_osk_wq_flush();
+ kfree(work_object);
+}
+
+void _mali_osk_wq_delete_work_nonflush(_mali_osk_wq_work_t *work)
+{
+ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work;
+ kfree(work_object);
+}
+
+void _mali_osk_wq_schedule_work(_mali_osk_wq_work_t *work)
+{
+ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work;
+#if MALI_LICENSE_IS_GPL
+ queue_work(mali_wq_normal, &work_object->work_handle);
+#else
+ schedule_work(&work_object->work_handle);
+#endif
+}
+
+void _mali_osk_wq_schedule_work_high_pri(_mali_osk_wq_work_t *work)
+{
+ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work;
+#if MALI_LICENSE_IS_GPL
+ queue_work(mali_wq_high, &work_object->work_handle);
+#else
+ schedule_work(&work_object->work_handle);
+#endif
+}
+
+static void _mali_osk_wq_work_func(struct work_struct *work)
+{
+ mali_osk_wq_work_object_t *work_object;
+
+ work_object = _MALI_OSK_CONTAINER_OF(work, mali_osk_wq_work_object_t, work_handle);
+
+#if MALI_LICENSE_IS_GPL
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36)
+ /* We want highest Dynamic priority of the thread so that the Jobs depending
+ ** on this thread could be scheduled in time. Without this, this thread might
+ ** sometimes need to wait for some threads in user mode to finish its round-robin
+ ** time, causing *bubble* in the Mali pipeline. Thanks to the new implementation
+ ** of high-priority workqueue in new kernel, this only happens in older kernel.
+ */
+ if (MALI_TRUE == work_object->high_pri) {
+ set_user_nice(current, -19);
+ }
+#endif
+#endif /* MALI_LICENSE_IS_GPL */
+
+ work_object->handler(work_object->data);
+}
+
+static void _mali_osk_wq_delayed_work_func(struct work_struct *work)
+{
+ mali_osk_wq_delayed_work_object_t *work_object;
+
+ work_object = _MALI_OSK_CONTAINER_OF(work, mali_osk_wq_delayed_work_object_t, work.work);
+ work_object->handler(work_object->data);
+}
+
+mali_osk_wq_delayed_work_object_t *_mali_osk_wq_delayed_create_work(_mali_osk_wq_work_handler_t handler, void *data)
+{
+ mali_osk_wq_delayed_work_object_t *work = kmalloc(sizeof(mali_osk_wq_delayed_work_object_t), GFP_KERNEL);
+
+ if (NULL == work) return NULL;
+
+ work->handler = handler;
+ work->data = data;
+
+ INIT_DELAYED_WORK(&work->work, _mali_osk_wq_delayed_work_func);
+
+ return work;
+}
+
+void _mali_osk_wq_delayed_delete_work_nonflush(_mali_osk_wq_delayed_work_t *work)
+{
+ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work;
+ kfree(work_object);
+}
+
+void _mali_osk_wq_delayed_cancel_work_async(_mali_osk_wq_delayed_work_t *work)
+{
+ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work;
+ cancel_delayed_work(&work_object->work);
+}
+
+void _mali_osk_wq_delayed_cancel_work_sync(_mali_osk_wq_delayed_work_t *work)
+{
+ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work;
+ cancel_delayed_work_sync(&work_object->work);
+}
+
+void _mali_osk_wq_delayed_schedule_work(_mali_osk_wq_delayed_work_t *work, u32 delay)
+{
+ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work;
+
+#if MALI_LICENSE_IS_GPL
+ queue_delayed_work(mali_wq_normal, &work_object->work, delay);
+#else
+ schedule_delayed_work(&work_object->work, delay);
+#endif
+
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_pmu_power_up_down.c b/drivers/gpu/arm/utgard/linux/mali_pmu_power_up_down.c
new file mode 100644
index 000000000000..61ff5c8fdca8
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_pmu_power_up_down.c
@@ -0,0 +1,23 @@
+/**
+ * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_pmu_power_up_down.c
+ */
+
+#include <linux/module.h>
+#include "mali_executor.h"
+
+int mali_perf_set_num_pp_cores(unsigned int num_cores)
+{
+ return mali_executor_set_perf_level(num_cores, MALI_FALSE);
+}
+
+EXPORT_SYMBOL(mali_perf_set_num_pp_cores);
diff --git a/drivers/gpu/arm/utgard/linux/mali_profiling_events.h b/drivers/gpu/arm/utgard/linux/mali_profiling_events.h
new file mode 100644
index 000000000000..0b90e8c5cf26
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_profiling_events.h
@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2012, 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_PROFILING_EVENTS_H__
+#define __MALI_PROFILING_EVENTS_H__
+
+/* Simple wrapper in order to find the OS specific location of this file */
+#include <linux/mali/mali_utgard_profiling_events.h>
+
+#endif /* __MALI_PROFILING_EVENTS_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_profiling_gator_api.h b/drivers/gpu/arm/utgard/linux/mali_profiling_gator_api.h
new file mode 100644
index 000000000000..c98d127366ba
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_profiling_gator_api.h
@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2012-2013, 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_PROFILING_GATOR_API_H__
+#define __MALI_PROFILING_GATOR_API_H__
+
+/* Simple wrapper in order to find the OS specific location of this file */
+#include <linux/mali/mali_utgard_profiling_gator_api.h>
+
+#endif /* __MALI_PROFILING_GATOR_API_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_profiling_internal.c b/drivers/gpu/arm/utgard/linux/mali_profiling_internal.c
new file mode 100644
index 000000000000..12aef4194ff5
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_profiling_internal.c
@@ -0,0 +1,275 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include "mali_kernel_common.h"
+#include "mali_osk.h"
+#include "mali_osk_mali.h"
+#include "mali_ukk.h"
+#include "mali_timestamp.h"
+#include "mali_osk_profiling.h"
+#include "mali_user_settings_db.h"
+#include "mali_profiling_internal.h"
+
+typedef struct mali_profiling_entry {
+ u64 timestamp;
+ u32 event_id;
+ u32 data[5];
+} mali_profiling_entry;
+
+typedef enum mali_profiling_state {
+ MALI_PROFILING_STATE_UNINITIALIZED,
+ MALI_PROFILING_STATE_IDLE,
+ MALI_PROFILING_STATE_RUNNING,
+ MALI_PROFILING_STATE_RETURN,
+} mali_profiling_state;
+
+static _mali_osk_mutex_t *lock = NULL;
+static mali_profiling_state prof_state = MALI_PROFILING_STATE_UNINITIALIZED;
+static mali_profiling_entry *profile_entries = NULL;
+static _mali_osk_atomic_t profile_insert_index;
+static u32 profile_mask = 0;
+
+static inline void add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4);
+
+void probe_mali_timeline_event(void *data, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned
+ int d2, unsigned int d3, unsigned int d4))
+{
+ add_event(event_id, d0, d1, d2, d3, d4);
+}
+
+_mali_osk_errcode_t _mali_internal_profiling_init(mali_bool auto_start)
+{
+ profile_entries = NULL;
+ profile_mask = 0;
+ _mali_osk_atomic_init(&profile_insert_index, 0);
+
+ lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_PROFILING);
+ if (NULL == lock) {
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ prof_state = MALI_PROFILING_STATE_IDLE;
+
+ if (MALI_TRUE == auto_start) {
+ u32 limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; /* Use maximum buffer size */
+
+ mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE);
+ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_start(&limit)) {
+ return _MALI_OSK_ERR_FAULT;
+ }
+ }
+
+ return _MALI_OSK_ERR_OK;
+}
+
+void _mali_internal_profiling_term(void)
+{
+ u32 count;
+
+ /* Ensure profiling is stopped */
+ _mali_internal_profiling_stop(&count);
+
+ prof_state = MALI_PROFILING_STATE_UNINITIALIZED;
+
+ if (NULL != profile_entries) {
+ _mali_osk_vfree(profile_entries);
+ profile_entries = NULL;
+ }
+
+ if (NULL != lock) {
+ _mali_osk_mutex_term(lock);
+ lock = NULL;
+ }
+}
+
+_mali_osk_errcode_t _mali_internal_profiling_start(u32 *limit)
+{
+ _mali_osk_errcode_t ret;
+ mali_profiling_entry *new_profile_entries;
+
+ _mali_osk_mutex_wait(lock);
+
+ if (MALI_PROFILING_STATE_RUNNING == prof_state) {
+ _mali_osk_mutex_signal(lock);
+ return _MALI_OSK_ERR_BUSY;
+ }
+
+ new_profile_entries = _mali_osk_valloc(*limit * sizeof(mali_profiling_entry));
+
+ if (NULL == new_profile_entries) {
+ _mali_osk_mutex_signal(lock);
+ _mali_osk_vfree(new_profile_entries);
+ return _MALI_OSK_ERR_NOMEM;
+ }
+
+ if (MALI_PROFILING_MAX_BUFFER_ENTRIES < *limit) {
+ *limit = MALI_PROFILING_MAX_BUFFER_ENTRIES;
+ }
+
+ profile_mask = 1;
+ while (profile_mask <= *limit) {
+ profile_mask <<= 1;
+ }
+ profile_mask >>= 1;
+
+ *limit = profile_mask;
+
+ profile_mask--; /* turns the power of two into a mask of one less */
+
+ if (MALI_PROFILING_STATE_IDLE != prof_state) {
+ _mali_osk_mutex_signal(lock);
+ _mali_osk_vfree(new_profile_entries);
+ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */
+ }
+
+ profile_entries = new_profile_entries;
+
+ ret = _mali_timestamp_reset();
+
+ if (_MALI_OSK_ERR_OK == ret) {
+ prof_state = MALI_PROFILING_STATE_RUNNING;
+ } else {
+ _mali_osk_vfree(profile_entries);
+ profile_entries = NULL;
+ }
+
+ register_trace_mali_timeline_event(probe_mali_timeline_event, NULL);
+
+ _mali_osk_mutex_signal(lock);
+ return ret;
+}
+
+static inline void add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4)
+{
+ u32 cur_index = (_mali_osk_atomic_inc_return(&profile_insert_index) - 1) & profile_mask;
+
+ profile_entries[cur_index].timestamp = _mali_timestamp_get();
+ profile_entries[cur_index].event_id = event_id;
+ profile_entries[cur_index].data[0] = data0;
+ profile_entries[cur_index].data[1] = data1;
+ profile_entries[cur_index].data[2] = data2;
+ profile_entries[cur_index].data[3] = data3;
+ profile_entries[cur_index].data[4] = data4;
+
+ /* If event is "leave API function", add current memory usage to the event
+ * as data point 4. This is used in timeline profiling to indicate how
+ * much memory was used when leaving a function. */
+ if (event_id == (MALI_PROFILING_EVENT_TYPE_SINGLE | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | MALI_PROFILING_EVENT_REASON_SINGLE_SW_LEAVE_API_FUNC)) {
+ profile_entries[cur_index].data[4] = _mali_ukk_report_memory_usage();
+ }
+}
+
+_mali_osk_errcode_t _mali_internal_profiling_stop(u32 *count)
+{
+ _mali_osk_mutex_wait(lock);
+
+ if (MALI_PROFILING_STATE_RUNNING != prof_state) {
+ _mali_osk_mutex_signal(lock);
+ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */
+ }
+
+ /* go into return state (user to retreive events), no more events will be added after this */
+ prof_state = MALI_PROFILING_STATE_RETURN;
+
+ unregister_trace_mali_timeline_event(probe_mali_timeline_event, NULL);
+
+ _mali_osk_mutex_signal(lock);
+
+ tracepoint_synchronize_unregister();
+
+ *count = _mali_osk_atomic_read(&profile_insert_index);
+ if (*count > profile_mask) *count = profile_mask;
+
+ return _MALI_OSK_ERR_OK;
+}
+
+u32 _mali_internal_profiling_get_count(void)
+{
+ u32 retval = 0;
+
+ _mali_osk_mutex_wait(lock);
+ if (MALI_PROFILING_STATE_RETURN == prof_state) {
+ retval = _mali_osk_atomic_read(&profile_insert_index);
+ if (retval > profile_mask) retval = profile_mask;
+ }
+ _mali_osk_mutex_signal(lock);
+
+ return retval;
+}
+
+_mali_osk_errcode_t _mali_internal_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5])
+{
+ u32 raw_index = _mali_osk_atomic_read(&profile_insert_index);
+
+ _mali_osk_mutex_wait(lock);
+
+ if (index < profile_mask) {
+ if ((raw_index & ~profile_mask) != 0) {
+ index += raw_index;
+ index &= profile_mask;
+ }
+
+ if (prof_state != MALI_PROFILING_STATE_RETURN) {
+ _mali_osk_mutex_signal(lock);
+ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */
+ }
+
+ if (index >= raw_index) {
+ _mali_osk_mutex_signal(lock);
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ *timestamp = profile_entries[index].timestamp;
+ *event_id = profile_entries[index].event_id;
+ data[0] = profile_entries[index].data[0];
+ data[1] = profile_entries[index].data[1];
+ data[2] = profile_entries[index].data[2];
+ data[3] = profile_entries[index].data[3];
+ data[4] = profile_entries[index].data[4];
+ } else {
+ _mali_osk_mutex_signal(lock);
+ return _MALI_OSK_ERR_FAULT;
+ }
+
+ _mali_osk_mutex_signal(lock);
+ return _MALI_OSK_ERR_OK;
+}
+
+_mali_osk_errcode_t _mali_internal_profiling_clear(void)
+{
+ _mali_osk_mutex_wait(lock);
+
+ if (MALI_PROFILING_STATE_RETURN != prof_state) {
+ _mali_osk_mutex_signal(lock);
+ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */
+ }
+
+ prof_state = MALI_PROFILING_STATE_IDLE;
+ profile_mask = 0;
+ _mali_osk_atomic_init(&profile_insert_index, 0);
+
+ if (NULL != profile_entries) {
+ _mali_osk_vfree(profile_entries);
+ profile_entries = NULL;
+ }
+
+ _mali_osk_mutex_signal(lock);
+ return _MALI_OSK_ERR_OK;
+}
+
+mali_bool _mali_internal_profiling_is_recording(void)
+{
+ return prof_state == MALI_PROFILING_STATE_RUNNING ? MALI_TRUE : MALI_FALSE;
+}
+
+mali_bool _mali_internal_profiling_have_recording(void)
+{
+ return prof_state == MALI_PROFILING_STATE_RETURN ? MALI_TRUE : MALI_FALSE;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_profiling_internal.h b/drivers/gpu/arm/utgard/linux/mali_profiling_internal.h
new file mode 100644
index 000000000000..1c6f4da691d2
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_profiling_internal.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_PROFILING_INTERNAL_H__
+#define __MALI_PROFILING_INTERNAL_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "mali_osk.h"
+
+int _mali_internal_profiling_init(mali_bool auto_start);
+void _mali_internal_profiling_term(void);
+
+mali_bool _mali_internal_profiling_is_recording(void);
+mali_bool _mali_internal_profiling_have_recording(void);
+_mali_osk_errcode_t _mali_internal_profiling_clear(void);
+_mali_osk_errcode_t _mali_internal_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]);
+u32 _mali_internal_profiling_get_count(void);
+int _mali_internal_profiling_stop(u32 *count);
+int _mali_internal_profiling_start(u32 *limit);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __MALI_PROFILING_INTERNAL_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_sync.c b/drivers/gpu/arm/utgard/linux/mali_sync.c
new file mode 100644
index 000000000000..dc1e3a2a4d73
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_sync.c
@@ -0,0 +1,447 @@
+/*
+ * Copyright (C) 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include "mali_sync.h"
+
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_timeline.h"
+#include "mali_executor.h"
+
+#include <linux/file.h>
+#include <linux/seq_file.h>
+#include <linux/module.h>
+#include <linux/fcntl.h>
+
+struct mali_sync_pt {
+ struct sync_pt sync_pt;
+ struct mali_sync_flag *flag;
+ struct sync_timeline *sync_tl; /**< Sync timeline this pt is connected to. */
+};
+
+/**
+ * The sync flag is used to connect sync fences to the Mali Timeline system. Sync fences can be
+ * created from a sync flag, and when the flag is signaled, the sync fences will also be signaled.
+ */
+struct mali_sync_flag {
+ struct sync_timeline *sync_tl; /**< Sync timeline this flag is connected to. */
+ u32 point; /**< Point on timeline. */
+ int status; /**< 0 if unsignaled, 1 if signaled without error or negative if signaled with error. */
+ struct kref refcount; /**< Reference count. */
+};
+
+/**
+ * Mali sync timeline is used to connect mali timeline to sync_timeline.
+ * When fence timeout can print more detailed mali timeline system info.
+ */
+struct mali_sync_timeline_container {
+ struct sync_timeline sync_timeline;
+ struct mali_timeline *timeline;
+};
+
+MALI_STATIC_INLINE struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt)
+{
+ return container_of(pt, struct mali_sync_pt, sync_pt);
+}
+
+MALI_STATIC_INLINE struct mali_sync_timeline_container *to_mali_sync_tl_container(struct sync_timeline *sync_tl)
+{
+ return container_of(sync_tl, struct mali_sync_timeline_container, sync_timeline);
+}
+
+static struct sync_pt *timeline_dup(struct sync_pt *pt)
+{
+ struct mali_sync_pt *mpt, *new_mpt;
+ struct sync_pt *new_pt;
+
+ MALI_DEBUG_ASSERT_POINTER(pt);
+ mpt = to_mali_sync_pt(pt);
+
+ new_pt = sync_pt_create(mpt->sync_tl, sizeof(struct mali_sync_pt));
+ if (NULL == new_pt) return NULL;
+
+ new_mpt = to_mali_sync_pt(new_pt);
+
+ mali_sync_flag_get(mpt->flag);
+ new_mpt->flag = mpt->flag;
+ new_mpt->sync_tl = mpt->sync_tl;
+
+ return new_pt;
+}
+
+static int timeline_has_signaled(struct sync_pt *pt)
+{
+ struct mali_sync_pt *mpt;
+
+ MALI_DEBUG_ASSERT_POINTER(pt);
+ mpt = to_mali_sync_pt(pt);
+
+ MALI_DEBUG_ASSERT_POINTER(mpt->flag);
+
+ return mpt->flag->status;
+}
+
+static int timeline_compare(struct sync_pt *pta, struct sync_pt *ptb)
+{
+ struct mali_sync_pt *mpta;
+ struct mali_sync_pt *mptb;
+ u32 a, b;
+
+ MALI_DEBUG_ASSERT_POINTER(pta);
+ MALI_DEBUG_ASSERT_POINTER(ptb);
+ mpta = to_mali_sync_pt(pta);
+ mptb = to_mali_sync_pt(ptb);
+
+ MALI_DEBUG_ASSERT_POINTER(mpta->flag);
+ MALI_DEBUG_ASSERT_POINTER(mptb->flag);
+
+ a = mpta->flag->point;
+ b = mptb->flag->point;
+
+ if (a == b) return 0;
+
+ return ((b - a) < (a - b) ? -1 : 1);
+}
+
+static void timeline_free_pt(struct sync_pt *pt)
+{
+ struct mali_sync_pt *mpt;
+
+ MALI_DEBUG_ASSERT_POINTER(pt);
+ mpt = to_mali_sync_pt(pt);
+
+ mali_sync_flag_put(mpt->flag);
+}
+
+static void timeline_release(struct sync_timeline *sync_timeline)
+{
+ struct mali_sync_timeline_container *mali_sync_tl = NULL;
+ struct mali_timeline *mali_tl = NULL;
+
+ MALI_DEBUG_ASSERT_POINTER(sync_timeline);
+
+ mali_sync_tl = to_mali_sync_tl_container(sync_timeline);
+ MALI_DEBUG_ASSERT_POINTER(mali_sync_tl);
+
+ mali_tl = mali_sync_tl->timeline;
+
+ /* always signaled timeline didn't have mali container */
+ if (mali_tl) {
+ if (NULL != mali_tl->spinlock) {
+ mali_spinlock_reentrant_term(mali_tl->spinlock);
+ }
+ _mali_osk_free(mali_tl);
+ }
+
+ module_put(THIS_MODULE);
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+static void timeline_print_pt(struct seq_file *s, struct sync_pt *sync_pt)
+{
+ struct mali_sync_pt *mpt;
+
+ MALI_DEBUG_ASSERT_POINTER(s);
+ MALI_DEBUG_ASSERT_POINTER(sync_pt);
+
+ mpt = to_mali_sync_pt(sync_pt);
+
+ /* It is possible this sync point is just under construct,
+ * make sure the flag is valid before accessing it
+ */
+ if (mpt->flag) {
+ seq_printf(s, "%u", mpt->flag->point);
+ } else {
+ seq_printf(s, "uninitialized");
+ }
+}
+
+static void timeline_print_obj(struct seq_file *s, struct sync_timeline *sync_tl)
+{
+ struct mali_sync_timeline_container *mali_sync_tl = NULL;
+ struct mali_timeline *mali_tl = NULL;
+
+ MALI_DEBUG_ASSERT_POINTER(sync_tl);
+
+ mali_sync_tl = to_mali_sync_tl_container(sync_tl);
+ MALI_DEBUG_ASSERT_POINTER(mali_sync_tl);
+
+ mali_tl = mali_sync_tl->timeline;
+
+ if (NULL != mali_tl) {
+ seq_printf(s, "oldest (%u) ", mali_tl->point_oldest);
+ seq_printf(s, "next (%u)", mali_tl->point_next);
+ seq_printf(s, "\n");
+
+#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
+ {
+ u32 tid = _mali_osk_get_tid();
+ struct mali_timeline_system *system = mali_tl->system;
+
+ mali_spinlock_reentrant_wait(mali_tl->spinlock, tid);
+ if (!mali_tl->destroyed) {
+ mali_spinlock_reentrant_wait(system->spinlock, tid);
+ mali_timeline_debug_print_timeline(mali_tl, s);
+ mali_spinlock_reentrant_signal(system->spinlock, tid);
+ }
+ mali_spinlock_reentrant_signal(mali_tl->spinlock, tid);
+
+ /* dump job queue status and group running status */
+ mali_executor_status_dump();
+ }
+#endif
+ }
+}
+#else
+static void timeline_pt_value_str(struct sync_pt *pt, char *str, int size)
+{
+ struct mali_sync_pt *mpt;
+
+ MALI_DEBUG_ASSERT_POINTER(str);
+ MALI_DEBUG_ASSERT_POINTER(pt);
+
+ mpt = to_mali_sync_pt(pt);
+
+ /* It is possible this sync point is just under construct,
+ * make sure the flag is valid before accessing it
+ */
+ if (mpt->flag) {
+ _mali_osk_snprintf(str, size, "%u", mpt->flag->point);
+ } else {
+ _mali_osk_snprintf(str, size, "uninitialized");
+ }
+}
+
+static void timeline_value_str(struct sync_timeline *timeline, char *str, int size)
+{
+ struct mali_sync_timeline_container *mali_sync_tl = NULL;
+ struct mali_timeline *mali_tl = NULL;
+
+ MALI_DEBUG_ASSERT_POINTER(timeline);
+
+ mali_sync_tl = to_mali_sync_tl_container(timeline);
+ MALI_DEBUG_ASSERT_POINTER(mali_sync_tl);
+
+ mali_tl = mali_sync_tl->timeline;
+
+ if (NULL != mali_tl) {
+ _mali_osk_snprintf(str, size, "oldest (%u) ", mali_tl->point_oldest);
+ _mali_osk_snprintf(str, size, "next (%u)", mali_tl->point_next);
+ _mali_osk_snprintf(str, size, "\n");
+
+#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
+ {
+ u32 tid = _mali_osk_get_tid();
+ struct mali_timeline_system *system = mali_tl->system;
+
+ mali_spinlock_reentrant_wait(mali_tl->spinlock, tid);
+ if (!mali_tl->destroyed) {
+ mali_spinlock_reentrant_wait(system->spinlock, tid);
+ mali_timeline_debug_direct_print_timeline(mali_tl);
+ mali_spinlock_reentrant_signal(system->spinlock, tid);
+ }
+ mali_spinlock_reentrant_signal(mali_tl->spinlock, tid);
+
+ /* dump job queue status and group running status */
+ mali_executor_status_dump();
+ }
+#endif
+ }
+}
+#endif
+
+
+static struct sync_timeline_ops mali_timeline_ops = {
+ .driver_name = "Mali",
+ .dup = timeline_dup,
+ .has_signaled = timeline_has_signaled,
+ .compare = timeline_compare,
+ .free_pt = timeline_free_pt,
+ .release_obj = timeline_release,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+ .print_pt = timeline_print_pt,
+ .print_obj = timeline_print_obj,
+#else
+ .pt_value_str = timeline_pt_value_str,
+ .timeline_value_str = timeline_value_str,
+#endif
+};
+
+struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name)
+{
+ struct sync_timeline *sync_tl;
+ struct mali_sync_timeline_container *mali_sync_tl;
+
+ sync_tl = sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline_container), name);
+ if (NULL == sync_tl) return NULL;
+
+ mali_sync_tl = to_mali_sync_tl_container(sync_tl);
+ mali_sync_tl->timeline = timeline;
+
+ /* Grab a reference on the module to ensure the callbacks are present
+ * as long some timeline exists. The reference is released when the
+ * timeline is freed.
+ * Since this function is called from a ioctl on an open file we know
+ * we already have a reference, so using __module_get is safe. */
+ __module_get(THIS_MODULE);
+
+ return sync_tl;
+}
+
+s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence)
+{
+ s32 fd = -1;
+
+ fd = get_unused_fd_flags(O_CLOEXEC);
+ if (fd < 0) {
+ sync_fence_put(sync_fence);
+ return -1;
+ }
+ sync_fence_install(sync_fence, fd);
+
+ return fd;
+}
+
+struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2)
+{
+ struct sync_fence *sync_fence;
+
+ MALI_DEBUG_ASSERT_POINTER(sync_fence1);
+ MALI_DEBUG_ASSERT_POINTER(sync_fence1);
+
+ sync_fence = sync_fence_merge("mali_merge_fence", sync_fence1, sync_fence2);
+ sync_fence_put(sync_fence1);
+ sync_fence_put(sync_fence2);
+
+ return sync_fence;
+}
+
+struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl)
+{
+ struct mali_sync_flag *flag;
+ struct sync_fence *sync_fence;
+
+ MALI_DEBUG_ASSERT_POINTER(sync_tl);
+
+ flag = mali_sync_flag_create(sync_tl, 0);
+ if (NULL == flag) return NULL;
+
+ sync_fence = mali_sync_flag_create_fence(flag);
+
+ mali_sync_flag_signal(flag, 0);
+ mali_sync_flag_put(flag);
+
+ return sync_fence;
+}
+
+struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, mali_timeline_point point)
+{
+ struct mali_sync_flag *flag;
+
+ if (NULL == sync_tl) return NULL;
+
+ flag = _mali_osk_calloc(1, sizeof(*flag));
+ if (NULL == flag) return NULL;
+
+ flag->sync_tl = sync_tl;
+ flag->point = point;
+
+ flag->status = 0;
+ kref_init(&flag->refcount);
+
+ return flag;
+}
+
+void mali_sync_flag_get(struct mali_sync_flag *flag)
+{
+ MALI_DEBUG_ASSERT_POINTER(flag);
+ kref_get(&flag->refcount);
+}
+
+/**
+ * Free sync flag.
+ *
+ * @param ref kref object embedded in sync flag that should be freed.
+ */
+static void mali_sync_flag_free(struct kref *ref)
+{
+ struct mali_sync_flag *flag;
+
+ MALI_DEBUG_ASSERT_POINTER(ref);
+ flag = container_of(ref, struct mali_sync_flag, refcount);
+
+ _mali_osk_free(flag);
+}
+
+void mali_sync_flag_put(struct mali_sync_flag *flag)
+{
+ MALI_DEBUG_ASSERT_POINTER(flag);
+ kref_put(&flag->refcount, mali_sync_flag_free);
+}
+
+void mali_sync_flag_signal(struct mali_sync_flag *flag, int error)
+{
+ MALI_DEBUG_ASSERT_POINTER(flag);
+
+ MALI_DEBUG_ASSERT(0 == flag->status);
+ flag->status = (0 > error) ? error : 1;
+
+ _mali_osk_write_mem_barrier();
+
+ sync_timeline_signal(flag->sync_tl);
+}
+
+/**
+ * Create a sync point attached to given sync flag.
+ *
+ * @note Sync points must be triggered in *exactly* the same order as they are created.
+ *
+ * @param flag Sync flag.
+ * @return New sync point if successful, NULL if not.
+ */
+static struct sync_pt *mali_sync_flag_create_pt(struct mali_sync_flag *flag)
+{
+ struct sync_pt *pt;
+ struct mali_sync_pt *mpt;
+
+ MALI_DEBUG_ASSERT_POINTER(flag);
+ MALI_DEBUG_ASSERT_POINTER(flag->sync_tl);
+
+ pt = sync_pt_create(flag->sync_tl, sizeof(struct mali_sync_pt));
+ if (NULL == pt) return NULL;
+
+ mali_sync_flag_get(flag);
+
+ mpt = to_mali_sync_pt(pt);
+ mpt->flag = flag;
+ mpt->sync_tl = flag->sync_tl;
+
+ return pt;
+}
+
+struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag)
+{
+ struct sync_pt *sync_pt;
+ struct sync_fence *sync_fence;
+
+ MALI_DEBUG_ASSERT_POINTER(flag);
+ MALI_DEBUG_ASSERT_POINTER(flag->sync_tl);
+
+ sync_pt = mali_sync_flag_create_pt(flag);
+ if (NULL == sync_pt) return NULL;
+
+ sync_fence = sync_fence_create("mali_flag_fence", sync_pt);
+ if (NULL == sync_fence) {
+ sync_pt_free(sync_pt);
+ return NULL;
+ }
+
+ return sync_fence;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_sync.h b/drivers/gpu/arm/utgard/linux/mali_sync.h
new file mode 100644
index 000000000000..0c541ff9a2e0
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_sync.h
@@ -0,0 +1,111 @@
+/*
+ * Copyright (C) 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_sync.h
+ *
+ * Mali interface for Linux sync objects.
+ */
+
+#ifndef _MALI_SYNC_H_
+#define _MALI_SYNC_H_
+
+#if defined(CONFIG_SYNC)
+
+#include <linux/seq_file.h>
+#include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0)
+#include <linux/sync.h>
+#else
+#include <sync.h>
+#endif
+
+
+#include "mali_osk.h"
+
+struct mali_sync_flag;
+struct mali_timeline;
+
+/**
+ * Create a sync timeline.
+ *
+ * @param name Name of the sync timeline.
+ * @return The new sync timeline if successful, NULL if not.
+ */
+struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name);
+
+/**
+ * Creates a file descriptor representing the sync fence. Will release sync fence if allocation of
+ * file descriptor fails.
+ *
+ * @param sync_fence Sync fence.
+ * @return File descriptor representing sync fence if successful, or -1 if not.
+ */
+s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence);
+
+/**
+ * Merges two sync fences. Both input sync fences will be released.
+ *
+ * @param sync_fence1 First sync fence.
+ * @param sync_fence2 Second sync fence.
+ * @return New sync fence that is the result of the merger if successful, or NULL if not.
+ */
+struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2);
+
+/**
+ * Create a sync fence that is already signaled.
+ *
+ * @param tl Sync timeline.
+ * @return New signaled sync fence if successful, NULL if not.
+ */
+struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl);
+
+/**
+ * Create a sync flag.
+ *
+ * @param sync_tl Sync timeline.
+ * @param point Point on Mali timeline.
+ * @return New sync flag if successful, NULL if not.
+ */
+struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, u32 point);
+
+/**
+ * Grab sync flag reference.
+ *
+ * @param flag Sync flag.
+ */
+void mali_sync_flag_get(struct mali_sync_flag *flag);
+
+/**
+ * Release sync flag reference. If this was the last reference, the sync flag will be freed.
+ *
+ * @param flag Sync flag.
+ */
+void mali_sync_flag_put(struct mali_sync_flag *flag);
+
+/**
+ * Signal sync flag. All sync fences created from this flag will be signaled.
+ *
+ * @param flag Sync flag to signal.
+ * @param error Negative error code, or 0 if no error.
+ */
+void mali_sync_flag_signal(struct mali_sync_flag *flag, int error);
+
+/**
+ * Create a sync fence attached to given sync flag.
+ *
+ * @param flag Sync flag.
+ * @return New sync fence if successful, NULL if not.
+ */
+struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag);
+
+#endif /* defined(CONFIG_SYNC) */
+
+#endif /* _MALI_SYNC_H_ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_uk_types.h b/drivers/gpu/arm/utgard/linux/mali_uk_types.h
new file mode 100644
index 000000000000..1884cdbba424
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_uk_types.h
@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2012, 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __MALI_UK_TYPES_H__
+#define __MALI_UK_TYPES_H__
+
+/* Simple wrapper in order to find the OS specific location of this file */
+#include <linux/mali/mali_utgard_uk_types.h>
+
+#endif /* __MALI_UK_TYPES_H__ */
diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_core.c b/drivers/gpu/arm/utgard/linux/mali_ukk_core.c
new file mode 100644
index 000000000000..41c1f48fc6c7
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_ukk_core.c
@@ -0,0 +1,146 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/fs.h> /* file system operations */
+#include <linux/slab.h> /* memort allocation functions */
+#include <asm/uaccess.h> /* user space access */
+
+#include "mali_ukk.h"
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_ukk_wrappers.h"
+
+int get_api_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_s __user *uargs)
+{
+ _mali_uk_get_api_version_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ if (0 != get_user(kargs.version, &uargs->version)) return -EFAULT;
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_get_api_version(&kargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT;
+ if (0 != put_user(kargs.compatible, &uargs->compatible)) return -EFAULT;
+
+ return 0;
+}
+
+int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_v2_s __user *uargs)
+{
+ _mali_uk_get_api_version_v2_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ if (0 != get_user(kargs.version, &uargs->version)) return -EFAULT;
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_get_api_version_v2(&kargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT;
+ if (0 != put_user(kargs.compatible, &uargs->compatible)) return -EFAULT;
+
+ return 0;
+}
+
+int wait_for_notification_wrapper(struct mali_session_data *session_data, _mali_uk_wait_for_notification_s __user *uargs)
+{
+ _mali_uk_wait_for_notification_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_wait_for_notification(&kargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ if (_MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS != kargs.type) {
+ kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */
+ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_wait_for_notification_s))) return -EFAULT;
+ } else {
+ if (0 != put_user(kargs.type, &uargs->type)) return -EFAULT;
+ }
+
+ return 0;
+}
+
+int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs)
+{
+ _mali_uk_post_notification_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ kargs.ctx = (uintptr_t)session_data;
+
+ if (0 != get_user(kargs.type, &uargs->type)) {
+ return -EFAULT;
+ }
+
+ err = _mali_ukk_post_notification(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ return 0;
+}
+
+int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs)
+{
+ _mali_uk_get_user_settings_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_get_user_settings(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ kargs.ctx = 0; /* prevent kernel address to be returned to user space */
+ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_get_user_settings_s))) return -EFAULT;
+
+ return 0;
+}
+
+int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs)
+{
+ _mali_uk_request_high_priority_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_request_high_priority(&kargs);
+
+ kargs.ctx = 0;
+
+ return map_errcode(err);
+}
+
+int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs)
+{
+ _mali_uk_pending_submit_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_pending_submit(&kargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ return 0;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_gp.c b/drivers/gpu/arm/utgard/linux/mali_ukk_gp.c
new file mode 100644
index 000000000000..d4144c0f5e48
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_ukk_gp.c
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/fs.h> /* file system operations */
+#include <asm/uaccess.h> /* user space access */
+
+#include "mali_ukk.h"
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_ukk_wrappers.h"
+
+int gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_gp_start_job_s __user *uargs)
+{
+ _mali_osk_errcode_t err;
+
+ /* If the job was started successfully, 0 is returned. If there was an error, but the job
+ * was started, we return -ENOENT. For anything else returned, the job was not started. */
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ err = _mali_ukk_gp_start_job(session_data, uargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ return 0;
+}
+
+int gp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_core_version_s __user *uargs)
+{
+ _mali_uk_get_gp_core_version_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_get_gp_core_version(&kargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ /* no known transactions to roll-back */
+
+ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT;
+
+ return 0;
+}
+
+int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk_gp_suspend_response_s __user *uargs)
+{
+ _mali_uk_gp_suspend_response_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_gp_suspend_response_s))) return -EFAULT;
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_gp_suspend_response(&kargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ if (0 != put_user(kargs.cookie, &uargs->cookie)) return -EFAULT;
+
+ /* no known transactions to roll-back */
+ return 0;
+}
+
+int gp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_number_of_cores_s __user *uargs)
+{
+ _mali_uk_get_gp_number_of_cores_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_get_gp_number_of_cores(&kargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ /* no known transactions to roll-back */
+
+ if (0 != put_user(kargs.number_of_cores, &uargs->number_of_cores)) return -EFAULT;
+
+ return 0;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_mem.c b/drivers/gpu/arm/utgard/linux/mali_ukk_mem.c
new file mode 100644
index 000000000000..ca1cba0585b3
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_ukk_mem.c
@@ -0,0 +1,333 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/fs.h> /* file system operations */
+#include <asm/uaccess.h> /* user space access */
+
+#include "mali_ukk.h"
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_ukk_wrappers.h"
+
+int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs)
+{
+ _mali_uk_alloc_mem_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_alloc_mem_s))) {
+ return -EFAULT;
+ }
+ kargs.ctx = (uintptr_t)session_data;
+
+ err = _mali_ukk_mem_allocate(&kargs);
+
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) {
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs)
+{
+ _mali_uk_free_mem_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_free_mem_s))) {
+ return -EFAULT;
+ }
+ kargs.ctx = (uintptr_t)session_data;
+
+ err = _mali_ukk_mem_free(&kargs);
+
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ if (0 != put_user(kargs.free_pages_nr, &uargs->free_pages_nr)) {
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs)
+{
+ _mali_uk_bind_mem_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_bind_mem_s))) {
+ return -EFAULT;
+ }
+ kargs.ctx = (uintptr_t)session_data;
+
+ err = _mali_ukk_mem_bind(&kargs);
+
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ return 0;
+}
+
+int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs)
+{
+ _mali_uk_unbind_mem_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_unbind_mem_s))) {
+ return -EFAULT;
+ }
+ kargs.ctx = (uintptr_t)session_data;
+
+ err = _mali_ukk_mem_unbind(&kargs);
+
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ return 0;
+}
+
+
+int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs)
+{
+ _mali_uk_cow_mem_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_mem_s))) {
+ return -EFAULT;
+ }
+ kargs.ctx = (uintptr_t)session_data;
+
+ err = _mali_ukk_mem_cow(&kargs);
+
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) {
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs)
+{
+ _mali_uk_cow_modify_range_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_modify_range_s))) {
+ return -EFAULT;
+ }
+ kargs.ctx = (uintptr_t)session_data;
+
+ err = _mali_ukk_mem_cow_modify_range(&kargs);
+
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ if (0 != put_user(kargs.change_pages_nr, &uargs->change_pages_nr)) {
+ return -EFAULT;
+ }
+ return 0;
+}
+
+
+int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs)
+{
+ _mali_uk_mem_resize_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_resize_s))) {
+ return -EFAULT;
+ }
+ kargs.ctx = (uintptr_t)session_data;
+
+ err = _mali_ukk_mem_resize(&kargs);
+
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ return 0;
+}
+
+int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs)
+{
+ _mali_uk_mem_write_safe_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_write_safe_s))) {
+ return -EFAULT;
+ }
+
+ kargs.ctx = (uintptr_t)session_data;
+
+ /* Check if we can access the buffers */
+ if (!access_ok(VERIFY_WRITE, kargs.dest, kargs.size)
+ || !access_ok(VERIFY_READ, kargs.src, kargs.size)) {
+ return -EINVAL;
+ }
+
+ /* Check if size wraps */
+ if ((kargs.size + kargs.dest) <= kargs.dest
+ || (kargs.size + kargs.src) <= kargs.src) {
+ return -EINVAL;
+ }
+
+ err = _mali_ukk_mem_write_safe(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ if (0 != put_user(kargs.size, &uargs->size)) {
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+
+
+int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs)
+{
+ _mali_uk_query_mmu_page_table_dump_size_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ kargs.ctx = (uintptr_t)session_data;
+
+ err = _mali_ukk_query_mmu_page_table_dump_size(&kargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ if (0 != put_user(kargs.size, &uargs->size)) return -EFAULT;
+
+ return 0;
+}
+
+int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs)
+{
+ _mali_uk_dump_mmu_page_table_s kargs;
+ _mali_osk_errcode_t err;
+ void __user *user_buffer;
+ void *buffer = NULL;
+ int rc = -EFAULT;
+
+ /* validate input */
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ /* the session_data pointer was validated by caller */
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_dump_mmu_page_table_s)))
+ goto err_exit;
+
+ user_buffer = (void __user *)(uintptr_t)kargs.buffer;
+ if (!access_ok(VERIFY_WRITE, user_buffer, kargs.size))
+ goto err_exit;
+
+ /* allocate temporary buffer (kernel side) to store mmu page table info */
+ if (kargs.size <= 0)
+ return -EINVAL;
+ /* Allow at most 8MiB buffers, this is more than enough to dump a fully
+ * populated page table. */
+ if (kargs.size > SZ_8M)
+ return -EINVAL;
+
+ buffer = (void *)(uintptr_t)_mali_osk_valloc(kargs.size);
+ if (NULL == buffer) {
+ rc = -ENOMEM;
+ goto err_exit;
+ }
+
+ kargs.ctx = (uintptr_t)session_data;
+ kargs.buffer = (uintptr_t)buffer;
+ err = _mali_ukk_dump_mmu_page_table(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ rc = map_errcode(err);
+ goto err_exit;
+ }
+
+ /* copy mmu page table info back to user space and update pointers */
+ if (0 != copy_to_user(user_buffer, buffer, kargs.size))
+ goto err_exit;
+
+ kargs.register_writes = kargs.register_writes -
+ (uintptr_t)buffer + (uintptr_t)user_buffer;
+ kargs.page_table_dump = kargs.page_table_dump -
+ (uintptr_t)buffer + (uintptr_t)user_buffer;
+
+ if (0 != copy_to_user(uargs, &kargs, sizeof(kargs)))
+ goto err_exit;
+
+ rc = 0;
+
+err_exit:
+ if (buffer) _mali_osk_vfree(buffer);
+ return rc;
+}
+
+int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs)
+{
+ _mali_osk_errcode_t err;
+ _mali_uk_profiling_memory_usage_get_s kargs;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) {
+ return -EFAULT;
+ }
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_mem_usage_get(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */
+ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) {
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_pp.c b/drivers/gpu/arm/utgard/linux/mali_ukk_pp.c
new file mode 100644
index 000000000000..4c1c381cb90f
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_ukk_pp.c
@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/fs.h> /* file system operations */
+#include <asm/uaccess.h> /* user space access */
+
+#include "mali_ukk.h"
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_ukk_wrappers.h"
+
+int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs)
+{
+ _mali_osk_errcode_t err;
+
+ /* If the job was started successfully, 0 is returned. If there was an error, but the job
+ * was started, we return -ENOENT. For anything else returned, the job was not started. */
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ err = _mali_ukk_pp_start_job(session_data, uargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ return 0;
+}
+
+int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs)
+{
+ _mali_osk_errcode_t err;
+
+ /* If the jobs were started successfully, 0 is returned. If there was an error, but the
+ * jobs were started, we return -ENOENT. For anything else returned, the jobs were not
+ * started. */
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ err = _mali_ukk_pp_and_gp_start_job(session_data, uargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ return 0;
+}
+
+int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs)
+{
+ _mali_uk_get_pp_number_of_cores_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ kargs.ctx = (uintptr_t)session_data;
+
+ err = _mali_ukk_get_pp_number_of_cores(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */
+ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_get_pp_number_of_cores_s))) {
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+int pp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_core_version_s __user *uargs)
+{
+ _mali_uk_get_pp_core_version_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_get_pp_core_version(&kargs);
+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
+
+ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT;
+
+ return 0;
+}
+
+int pp_disable_wb_wrapper(struct mali_session_data *session_data, _mali_uk_pp_disable_wb_s __user *uargs)
+{
+ _mali_uk_pp_disable_wb_s kargs;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session_data, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_pp_disable_wb_s))) return -EFAULT;
+
+ kargs.ctx = (uintptr_t)session_data;
+ _mali_ukk_pp_job_disable_wb(&kargs);
+
+ return 0;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_profiling.c b/drivers/gpu/arm/utgard/linux/mali_ukk_profiling.c
new file mode 100644
index 000000000000..e84544dee07d
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_ukk_profiling.c
@@ -0,0 +1,177 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/fs.h> /* file system operations */
+#include <asm/uaccess.h> /* user space access */
+#include <linux/slab.h>
+
+#include "mali_ukk.h"
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_ukk_wrappers.h"
+
+int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs)
+{
+ _mali_uk_profiling_add_event_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_add_event_s))) {
+ return -EFAULT;
+ }
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_profiling_add_event(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ return 0;
+}
+
+int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs)
+{
+ _mali_uk_sw_counters_report_s kargs;
+ _mali_osk_errcode_t err;
+ u32 *counter_buffer;
+ u32 __user *counters;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_sw_counters_report_s))) {
+ return -EFAULT;
+ }
+
+ /* make sure that kargs.num_counters is [at least somewhat] sane */
+ if (kargs.num_counters > 10000) {
+ MALI_DEBUG_PRINT(1, ("User space attempted to allocate too many counters.\n"));
+ return -EINVAL;
+ }
+
+ counter_buffer = (u32 *)kmalloc(sizeof(u32) * kargs.num_counters, GFP_KERNEL);
+ if (NULL == counter_buffer) {
+ return -ENOMEM;
+ }
+
+ counters = (u32 *)(uintptr_t)kargs.counters;
+
+ if (0 != copy_from_user(counter_buffer, counters, sizeof(u32) * kargs.num_counters)) {
+ kfree(counter_buffer);
+ return -EFAULT;
+ }
+
+ kargs.ctx = (uintptr_t)session_data;
+ kargs.counters = (uintptr_t)counter_buffer;
+
+ err = _mali_ukk_sw_counters_report(&kargs);
+
+ kfree(counter_buffer);
+
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ return 0;
+}
+
+int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs)
+{
+ _mali_uk_profiling_stream_fd_get_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) {
+ return -EFAULT;
+ }
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_profiling_stream_fd_get(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) {
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs)
+{
+ _mali_uk_profiling_control_set_s kargs;
+ _mali_osk_errcode_t err;
+ u8 *kernel_control_data = NULL;
+ u8 *kernel_response_data = NULL;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ if (0 != get_user(kargs.control_packet_size, &uargs->control_packet_size)) return -EFAULT;
+ if (0 != get_user(kargs.response_packet_size, &uargs->response_packet_size)) return -EFAULT;
+
+ kargs.ctx = (uintptr_t)session_data;
+
+ if (0 != kargs.control_packet_size) {
+
+ kernel_control_data = _mali_osk_calloc(1, kargs.control_packet_size);
+ if (NULL == kernel_control_data) {
+ return -ENOMEM;
+ }
+
+ MALI_DEBUG_ASSERT(0 != kargs.response_packet_size);
+
+ kernel_response_data = _mali_osk_calloc(1, kargs.response_packet_size);
+ if (NULL == kernel_response_data) {
+ _mali_osk_free(kernel_control_data);
+ return -ENOMEM;
+ }
+
+ kargs.control_packet_data = (uintptr_t)kernel_control_data;
+ kargs.response_packet_data = (uintptr_t)kernel_response_data;
+
+ if (0 != copy_from_user((void *)(uintptr_t)kernel_control_data, (void *)(uintptr_t)uargs->control_packet_data, kargs.control_packet_size)) {
+ _mali_osk_free(kernel_control_data);
+ _mali_osk_free(kernel_response_data);
+ return -EFAULT;
+ }
+
+ err = _mali_ukk_profiling_control_set(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ _mali_osk_free(kernel_control_data);
+ _mali_osk_free(kernel_response_data);
+ return map_errcode(err);
+ }
+
+ if (0 != kargs.response_packet_size && 0 != copy_to_user(((void *)(uintptr_t)uargs->response_packet_data), ((void *)(uintptr_t)kargs.response_packet_data), kargs.response_packet_size)) {
+ _mali_osk_free(kernel_control_data);
+ _mali_osk_free(kernel_response_data);
+ return -EFAULT;
+ }
+
+ if (0 != put_user(kargs.response_packet_size, &uargs->response_packet_size)) {
+ _mali_osk_free(kernel_control_data);
+ _mali_osk_free(kernel_response_data);
+ return -EFAULT;
+ }
+
+ _mali_osk_free(kernel_control_data);
+ _mali_osk_free(kernel_response_data);
+ } else {
+
+ err = _mali_ukk_profiling_control_set(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ }
+ return 0;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_soft_job.c b/drivers/gpu/arm/utgard/linux/mali_ukk_soft_job.c
new file mode 100644
index 000000000000..11c70060e489
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_ukk_soft_job.c
@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/fs.h> /* file system operations */
+#include <asm/uaccess.h> /* user space access */
+
+#include "mali_ukk.h"
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_ukk_wrappers.h"
+
+#include "mali_soft_job.h"
+#include "mali_timeline.h"
+
+int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs)
+{
+ _mali_uk_soft_job_start_s kargs;
+ u32 type, point;
+ u64 user_job;
+ struct mali_timeline_fence fence;
+ struct mali_soft_job *job = NULL;
+ u32 __user *job_id_ptr = NULL;
+
+ /* If the job was started successfully, 0 is returned. If there was an error, but the job
+ * was started, we return -ENOENT. For anything else returned, the job was not started. */
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+ MALI_CHECK_NON_NULL(session, -EINVAL);
+
+ MALI_DEBUG_ASSERT_POINTER(session->soft_job_system);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(kargs))) {
+ return -EFAULT;
+ }
+
+ type = kargs.type;
+ user_job = kargs.user_job;
+ job_id_ptr = (u32 __user *)(uintptr_t)kargs.job_id_ptr;
+
+ mali_timeline_fence_copy_uk_fence(&fence, &kargs.fence);
+
+ if ((MALI_SOFT_JOB_TYPE_USER_SIGNALED != type) && (MALI_SOFT_JOB_TYPE_SELF_SIGNALED != type)) {
+ MALI_DEBUG_PRINT_ERROR(("Invalid soft job type specified\n"));
+ return -EINVAL;
+ }
+
+ /* Create soft job. */
+ job = mali_soft_job_create(session->soft_job_system, (enum mali_soft_job_type)type, user_job);
+ if (unlikely(NULL == job)) {
+ return map_errcode(_MALI_OSK_ERR_NOMEM);
+ }
+
+ /* Write job id back to user space. */
+ if (0 != put_user(job->id, job_id_ptr)) {
+ MALI_PRINT_ERROR(("Mali Soft Job: failed to put job id"));
+ mali_soft_job_destroy(job);
+ return map_errcode(_MALI_OSK_ERR_NOMEM);
+ }
+
+ /* Start soft job. */
+ point = mali_soft_job_start(job, &fence);
+
+ if (0 != put_user(point, &uargs->point)) {
+ /* Let user space know that something failed after the job was started. */
+ return -ENOENT;
+ }
+
+ return 0;
+}
+
+int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs)
+{
+ u32 job_id;
+ _mali_osk_errcode_t err;
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ if (0 != get_user(job_id, &uargs->job_id)) return -EFAULT;
+
+ err = mali_soft_job_system_signal_job(session->soft_job_system, job_id);
+
+ return map_errcode(err);
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_timeline.c b/drivers/gpu/arm/utgard/linux/mali_ukk_timeline.c
new file mode 100644
index 000000000000..484d4041c869
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_ukk_timeline.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2013, 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/fs.h> /* file system operations */
+#include <asm/uaccess.h> /* user space access */
+
+#include "mali_ukk.h"
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_ukk_wrappers.h"
+
+#include "mali_timeline.h"
+#include "mali_timeline_fence_wait.h"
+#include "mali_timeline_sync_fence.h"
+
+int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs)
+{
+ u32 val;
+ mali_timeline_id timeline;
+ mali_timeline_point point;
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ if (0 != get_user(val, &uargs->timeline)) return -EFAULT;
+
+ if (MALI_UK_TIMELINE_MAX <= val) {
+ return -EINVAL;
+ }
+
+ timeline = (mali_timeline_id)val;
+
+ point = mali_timeline_system_get_latest_point(session->timeline_system, timeline);
+
+ if (0 != put_user(point, &uargs->point)) return -EFAULT;
+
+ return 0;
+}
+
+int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs)
+{
+ u32 timeout, status;
+ mali_bool ret;
+ _mali_uk_fence_t uk_fence;
+ struct mali_timeline_fence fence;
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ if (0 != copy_from_user(&uk_fence, &uargs->fence, sizeof(_mali_uk_fence_t))) return -EFAULT;
+ if (0 != get_user(timeout, &uargs->timeout)) return -EFAULT;
+
+ mali_timeline_fence_copy_uk_fence(&fence, &uk_fence);
+
+ ret = mali_timeline_fence_wait(session->timeline_system, &fence, timeout);
+ status = (MALI_TRUE == ret ? 1 : 0);
+
+ if (0 != put_user(status, &uargs->status)) return -EFAULT;
+
+ return 0;
+}
+
+int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs)
+{
+ s32 sync_fd = -1;
+ _mali_uk_fence_t uk_fence;
+ struct mali_timeline_fence fence;
+
+ MALI_DEBUG_ASSERT_POINTER(session);
+
+ if (0 != copy_from_user(&uk_fence, &uargs->fence, sizeof(_mali_uk_fence_t))) return -EFAULT;
+ mali_timeline_fence_copy_uk_fence(&fence, &uk_fence);
+
+#if defined(CONFIG_SYNC)
+ sync_fd = mali_timeline_sync_fence_create(session->timeline_system, &fence);
+#else
+ sync_fd = -1;
+#endif /* defined(CONFIG_SYNC) */
+
+ if (0 != put_user(sync_fd, &uargs->sync_fd)) return -EFAULT;
+
+ return 0;
+}
diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_vsync.c b/drivers/gpu/arm/utgard/linux/mali_ukk_vsync.c
new file mode 100644
index 000000000000..487c2478df4d
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_ukk_vsync.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+#include <linux/fs.h> /* file system operations */
+#include <asm/uaccess.h> /* user space access */
+
+#include "mali_ukk.h"
+#include "mali_osk.h"
+#include "mali_kernel_common.h"
+#include "mali_session.h"
+#include "mali_ukk_wrappers.h"
+
+
+int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs)
+{
+ _mali_uk_vsync_event_report_s kargs;
+ _mali_osk_errcode_t err;
+
+ MALI_CHECK_NON_NULL(uargs, -EINVAL);
+
+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_vsync_event_report_s))) {
+ return -EFAULT;
+ }
+
+ kargs.ctx = (uintptr_t)session_data;
+ err = _mali_ukk_vsync_event_report(&kargs);
+ if (_MALI_OSK_ERR_OK != err) {
+ return map_errcode(err);
+ }
+
+ return 0;
+}
+
diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_wrappers.h b/drivers/gpu/arm/utgard/linux/mali_ukk_wrappers.h
new file mode 100644
index 000000000000..ac84e446bd12
--- /dev/null
+++ b/drivers/gpu/arm/utgard/linux/mali_ukk_wrappers.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained from Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file mali_ukk_wrappers.h
+ * Defines the wrapper functions for each user-kernel function
+ */
+
+#ifndef __MALI_UKK_WRAPPERS_H__
+#define __MALI_UKK_WRAPPERS_H__
+
+#include "mali_uk_types.h"
+#include "mali_osk.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int wait_for_notification_wrapper(struct mali_session_data *session_data, _mali_uk_wait_for_notification_s __user *uargs);
+int get_api_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_s __user *uargs);
+int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_v2_s __user *uargs);
+int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs);
+int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs);
+int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs);
+int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs);
+
+int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs);
+int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs);
+int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs);
+int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs);
+int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs);
+int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs);
+int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs);
+int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs);
+int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs);
+int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs);
+int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs);
+
+int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs);
+int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs);
+int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs);
+int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs);
+int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs);
+int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs);
+int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs);
+int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs);
+int pp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_core_version_s __user *uargs);
+int pp_disable_wb_wrapper(struct mali_session_data *session_data, _mali_uk_pp_disable_wb_s __user *uargs);
+int gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_gp_start_job_s __user *uargs);
+int gp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_number_of_cores_s __user *uargs);
+int gp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_core_version_s __user *uargs);
+int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk_gp_suspend_response_s __user *uargs);
+
+int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs);
+int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs);
+int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs);
+int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs);
+
+int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs);
+
+
+int map_errcode(_mali_osk_errcode_t err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __MALI_UKK_WRAPPERS_H__ */