aboutsummaryrefslogtreecommitdiff
path: root/arch/arm/vfp/vfpmodule.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/vfp/vfpmodule.c')
-rw-r--r--arch/arm/vfp/vfpmodule.c133
1 files changed, 75 insertions, 58 deletions
diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c
index f25e7ec89416..79bcb4316930 100644
--- a/arch/arm/vfp/vfpmodule.c
+++ b/arch/arm/vfp/vfpmodule.c
@@ -33,7 +33,6 @@ void vfp_support_entry(void);
void vfp_null_entry(void);
void (*vfp_vector)(void) = vfp_null_entry;
-union vfp_state *last_VFP_context[NR_CPUS];
/*
* Dual-use variable.
@@ -43,6 +42,46 @@ union vfp_state *last_VFP_context[NR_CPUS];
unsigned int VFP_arch;
/*
+ * The pointer to the vfpstate structure of the thread which currently
+ * owns the context held in the VFP hardware, or NULL if the hardware
+ * context is invalid.
+ *
+ * For UP, this is sufficient to tell which thread owns the VFP context.
+ * However, for SMP, we also need to check the CPU number stored in the
+ * saved state too to catch migrations.
+ */
+union vfp_state *vfp_current_hw_state[NR_CPUS];
+
+/*
+ * Is 'thread's most up to date state stored in this CPUs hardware?
+ * Must be called from non-preemptible context.
+ */
+static bool vfp_state_in_hw(unsigned int cpu, struct thread_info *thread)
+{
+#ifdef CONFIG_SMP
+ if (thread->vfpstate.hard.cpu != cpu)
+ return false;
+#endif
+ return vfp_current_hw_state[cpu] == &thread->vfpstate;
+}
+
+/*
+ * Force a reload of the VFP context from the thread structure. We do
+ * this by ensuring that access to the VFP hardware is disabled, and
+ * clear last_VFP_context. Must be called from non-preemptible context.
+ */
+static void vfp_force_reload(unsigned int cpu, struct thread_info *thread)
+{
+ if (vfp_state_in_hw(cpu, thread)) {
+ fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
+ vfp_current_hw_state[cpu] = NULL;
+ }
+#ifdef CONFIG_SMP
+ thread->vfpstate.hard.cpu = NR_CPUS;
+#endif
+}
+
+/*
* Per-thread VFP initialization.
*/
static void vfp_thread_flush(struct thread_info *thread)
@@ -50,21 +89,27 @@ static void vfp_thread_flush(struct thread_info *thread)
union vfp_state *vfp = &thread->vfpstate;
unsigned int cpu;
- memset(vfp, 0, sizeof(union vfp_state));
-
- vfp->hard.fpexc = FPEXC_EN;
- vfp->hard.fpscr = FPSCR_ROUND_NEAREST;
-
/*
* Disable VFP to ensure we initialize it first. We must ensure
- * that the modification of last_VFP_context[] and hardware disable
- * are done for the same CPU and without preemption.
+ * that the modification of vfp_current_hw_state[] and hardware
+ * disable are done for the same CPU and without preemption.
+ *
+ * Do this first to ensure that preemption won't overwrite our
+ * state saving should access to the VFP be enabled at this point.
*/
cpu = get_cpu();
- if (last_VFP_context[cpu] == vfp)
- last_VFP_context[cpu] = NULL;
+ if (vfp_current_hw_state[cpu] == vfp)
+ vfp_current_hw_state[cpu] = NULL;
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
put_cpu();
+
+ memset(vfp, 0, sizeof(union vfp_state));
+
+ vfp->hard.fpexc = FPEXC_EN;
+ vfp->hard.fpscr = FPSCR_ROUND_NEAREST;
+#ifdef CONFIG_SMP
+ vfp->hard.cpu = NR_CPUS;
+#endif
}
static void vfp_thread_exit(struct thread_info *thread)
@@ -73,8 +118,8 @@ static void vfp_thread_exit(struct thread_info *thread)
union vfp_state *vfp = &thread->vfpstate;
unsigned int cpu = get_cpu();
- if (last_VFP_context[cpu] == vfp)
- last_VFP_context[cpu] = NULL;
+ if (vfp_current_hw_state[cpu] == vfp)
+ vfp_current_hw_state[cpu] = NULL;
put_cpu();
}
@@ -84,6 +129,9 @@ static void vfp_thread_copy(struct thread_info *thread)
vfp_sync_hwstate(parent);
thread->vfpstate = parent->vfpstate;
+#ifdef CONFIG_SMP
+ thread->vfpstate.hard.cpu = NR_CPUS;
+#endif
}
/*
@@ -129,17 +177,8 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
- if ((fpexc & FPEXC_EN) && last_VFP_context[cpu]) {
- vfp_save_state(last_VFP_context[cpu], fpexc);
- last_VFP_context[cpu]->hard.cpu = cpu;
- }
- /*
- * Thread migration, just force the reloading of the
- * state on the new CPU in case the VFP registers
- * contain stale data.
- */
- if (thread->vfpstate.hard.cpu != cpu)
- last_VFP_context[cpu] = NULL;
+ if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu])
+ vfp_save_state(vfp_current_hw_state[cpu], fpexc);
#endif
/*
@@ -415,7 +454,7 @@ static int vfp_pm_suspend(void)
}
/* clear any information we had about last context state */
- memset(last_VFP_context, 0, sizeof(last_VFP_context));
+ memset(vfp_current_hw_state, 0, sizeof(vfp_current_hw_state));
return 0;
}
@@ -443,15 +482,15 @@ static void vfp_pm_init(void)
static inline void vfp_pm_init(void) { }
#endif /* CONFIG_PM */
+/*
+ * Ensure that the VFP state stored in 'thread->vfpstate' is up to date
+ * with the hardware state.
+ */
void vfp_sync_hwstate(struct thread_info *thread)
{
unsigned int cpu = get_cpu();
- /*
- * If the thread we're interested in is the current owner of the
- * hardware VFP state, then we need to save its state.
- */
- if (last_VFP_context[cpu] == &thread->vfpstate) {
+ if (vfp_state_in_hw(cpu, thread)) {
u32 fpexc = fmrx(FPEXC);
/*
@@ -465,36 +504,13 @@ void vfp_sync_hwstate(struct thread_info *thread)
put_cpu();
}
+/* Ensure that the thread reloads the hardware VFP state on the next use. */
void vfp_flush_hwstate(struct thread_info *thread)
{
unsigned int cpu = get_cpu();
- /*
- * If the thread we're interested in is the current owner of the
- * hardware VFP state, then we need to save its state.
- */
- if (last_VFP_context[cpu] == &thread->vfpstate) {
- u32 fpexc = fmrx(FPEXC);
+ vfp_force_reload(cpu, thread);
- fmxr(FPEXC, fpexc & ~FPEXC_EN);
-
- /*
- * Set the context to NULL to force a reload the next time
- * the thread uses the VFP.
- */
- last_VFP_context[cpu] = NULL;
- }
-
-#ifdef CONFIG_SMP
- /*
- * For SMP we still have to take care of the case where the thread
- * migrates to another CPU and then back to the original CPU on which
- * the last VFP user is still the same thread. Mark the thread VFP
- * state as belonging to a non-existent CPU so that the saved one will
- * be reloaded in the above case.
- */
- thread->vfpstate.hard.cpu = NR_CPUS;
-#endif
put_cpu();
}
@@ -513,8 +529,7 @@ static int vfp_hotplug(struct notifier_block *b, unsigned long action,
void *hcpu)
{
if (action == CPU_DYING || action == CPU_DYING_FROZEN) {
- unsigned int cpu = (long)hcpu;
- last_VFP_context[cpu] = NULL;
+ vfp_force_reload((long)hcpu, current_thread_info());
} else if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
vfp_enable(NULL);
return NOTIFY_OK;
@@ -582,7 +597,6 @@ static int __init vfp_init(void)
elf_hwcap |= HWCAP_VFPv3D16;
}
#endif
-#ifdef CONFIG_NEON
/*
* Check for the presence of the Advanced SIMD
* load/store instructions, integer and single
@@ -590,10 +604,13 @@ static int __init vfp_init(void)
* for NEON if the hardware has the MVFR registers.
*/
if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) {
+#ifdef CONFIG_NEON
if ((fmrx(MVFR1) & 0x000fff00) == 0x00011100)
elf_hwcap |= HWCAP_NEON;
- }
#endif
+ if ((fmrx(MVFR1) & 0xf0000000) == 0x10000000)
+ elf_hwcap |= HWCAP_VFPv4;
+ }
}
return 0;
}