2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* linux/arch/arm/vfp/vfpmodule.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 2004 ARM Limited.
|
|
|
|
* Written by Deep Blue Solutions Limited.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*/
|
|
|
|
#include <linux/types.h>
|
2010-12-18 11:59:49 +01:00
|
|
|
#include <linux/cpu.h>
|
2011-02-10 11:08:32 +01:00
|
|
|
#include <linux/cpu_pm.h>
|
2012-05-11 18:42:37 +02:00
|
|
|
#include <linux/hardirq.h>
|
2005-04-17 00:20:36 +02:00
|
|
|
#include <linux/kernel.h>
|
2010-12-18 11:59:49 +01:00
|
|
|
#include <linux/notifier.h>
|
2005-04-17 00:20:36 +02:00
|
|
|
#include <linux/signal.h>
|
|
|
|
#include <linux/sched.h>
|
2010-12-18 11:59:49 +01:00
|
|
|
#include <linux/smp.h>
|
2005-04-17 00:20:36 +02:00
|
|
|
#include <linux/init.h>
|
2012-04-23 16:38:28 +02:00
|
|
|
#include <linux/uaccess.h>
|
|
|
|
#include <linux/user.h>
|
2013-05-16 11:41:48 +02:00
|
|
|
#include <linux/export.h>
|
2006-06-21 14:31:52 +02:00
|
|
|
|
2012-03-28 19:30:01 +02:00
|
|
|
#include <asm/cp15.h>
|
2010-07-01 14:41:05 +02:00
|
|
|
#include <asm/cputype.h>
|
2012-03-28 19:30:01 +02:00
|
|
|
#include <asm/system_info.h>
|
2006-06-21 14:31:52 +02:00
|
|
|
#include <asm/thread_notify.h>
|
2005-04-17 00:20:36 +02:00
|
|
|
#include <asm/vfp.h>
|
|
|
|
|
|
|
|
#include "vfpinstr.h"
|
|
|
|
#include "vfp.h"
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Our undef handlers (in entry.S)
|
|
|
|
*/
|
|
|
|
void vfp_testing_entry(void);
|
|
|
|
void vfp_support_entry(void);
|
2007-06-10 13:22:20 +02:00
|
|
|
void vfp_null_entry(void);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-06-10 13:22:20 +02:00
|
|
|
void (*vfp_vector)(void) = vfp_null_entry;
|
2011-07-09 14:44:04 +02:00
|
|
|
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
/*
|
|
|
|
* Dual-use variable.
|
|
|
|
* Used in startup: set to non-zero if VFP checks fail
|
|
|
|
* After startup, holds VFP architecture
|
|
|
|
*/
|
|
|
|
unsigned int VFP_arch;
|
|
|
|
|
2011-07-09 14:44:04 +02:00
|
|
|
/*
|
|
|
|
* The pointer to the vfpstate structure of the thread which currently
|
|
|
|
* owns the context held in the VFP hardware, or NULL if the hardware
|
|
|
|
* context is invalid.
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
*
|
|
|
|
* For UP, this is sufficient to tell which thread owns the VFP context.
|
|
|
|
* However, for SMP, we also need to check the CPU number stored in the
|
|
|
|
* saved state too to catch migrations.
|
2011-07-09 14:44:04 +02:00
|
|
|
*/
|
|
|
|
union vfp_state *vfp_current_hw_state[NR_CPUS];
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
/*
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
* Is 'thread's most up to date state stored in this CPUs hardware?
|
|
|
|
* Must be called from non-preemptible context.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
static bool vfp_state_in_hw(unsigned int cpu, struct thread_info *thread)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
if (thread->vfpstate.hard.cpu != cpu)
|
|
|
|
return false;
|
|
|
|
#endif
|
|
|
|
return vfp_current_hw_state[cpu] == &thread->vfpstate;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Force a reload of the VFP context from the thread structure. We do
|
|
|
|
* this by ensuring that access to the VFP hardware is disabled, and
|
2011-09-02 17:42:36 +02:00
|
|
|
* clear vfp_current_hw_state. Must be called from non-preemptible context.
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
*/
|
|
|
|
static void vfp_force_reload(unsigned int cpu, struct thread_info *thread)
|
|
|
|
{
|
|
|
|
if (vfp_state_in_hw(cpu, thread)) {
|
|
|
|
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
|
|
|
|
vfp_current_hw_state[cpu] = NULL;
|
|
|
|
}
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
thread->vfpstate.hard.cpu = NR_CPUS;
|
|
|
|
#endif
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2009-12-12 15:47:40 +01:00
|
|
|
/*
|
|
|
|
* Per-thread VFP initialization.
|
|
|
|
*/
|
|
|
|
static void vfp_thread_flush(struct thread_info *thread)
|
|
|
|
{
|
|
|
|
union vfp_state *vfp = &thread->vfpstate;
|
|
|
|
unsigned int cpu;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Disable VFP to ensure we initialize it first. We must ensure
|
2011-07-09 18:41:33 +02:00
|
|
|
* that the modification of vfp_current_hw_state[] and hardware
|
|
|
|
* disable are done for the same CPU and without preemption.
|
|
|
|
*
|
|
|
|
* Do this first to ensure that preemption won't overwrite our
|
|
|
|
* state saving should access to the VFP be enabled at this point.
|
2009-12-12 15:47:40 +01:00
|
|
|
*/
|
|
|
|
cpu = get_cpu();
|
2011-07-09 14:44:04 +02:00
|
|
|
if (vfp_current_hw_state[cpu] == vfp)
|
|
|
|
vfp_current_hw_state[cpu] = NULL;
|
2009-12-12 15:47:40 +01:00
|
|
|
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
|
|
|
|
put_cpu();
|
2011-07-09 18:41:33 +02:00
|
|
|
|
|
|
|
memset(vfp, 0, sizeof(union vfp_state));
|
|
|
|
|
|
|
|
vfp->hard.fpexc = FPEXC_EN;
|
|
|
|
vfp->hard.fpscr = FPSCR_ROUND_NEAREST;
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
vfp->hard.cpu = NR_CPUS;
|
|
|
|
#endif
|
2009-12-12 15:47:40 +01:00
|
|
|
}
|
|
|
|
|
2009-12-18 15:34:43 +01:00
|
|
|
static void vfp_thread_exit(struct thread_info *thread)
|
2009-12-12 15:47:40 +01:00
|
|
|
{
|
|
|
|
/* release case: Per-thread VFP cleanup. */
|
|
|
|
union vfp_state *vfp = &thread->vfpstate;
|
2009-12-18 15:34:43 +01:00
|
|
|
unsigned int cpu = get_cpu();
|
2009-12-12 15:47:40 +01:00
|
|
|
|
2011-07-09 14:44:04 +02:00
|
|
|
if (vfp_current_hw_state[cpu] == vfp)
|
|
|
|
vfp_current_hw_state[cpu] = NULL;
|
2009-12-18 15:34:43 +01:00
|
|
|
put_cpu();
|
2009-12-12 15:47:40 +01:00
|
|
|
}
|
|
|
|
|
2011-04-06 17:17:17 +02:00
|
|
|
static void vfp_thread_copy(struct thread_info *thread)
|
|
|
|
{
|
|
|
|
struct thread_info *parent = current_thread_info();
|
|
|
|
|
|
|
|
vfp_sync_hwstate(parent);
|
|
|
|
thread->vfpstate = parent->vfpstate;
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
thread->vfpstate.hard.cpu = NR_CPUS;
|
|
|
|
#endif
|
2011-04-06 17:17:17 +02:00
|
|
|
}
|
|
|
|
|
2009-12-12 15:47:40 +01:00
|
|
|
/*
|
|
|
|
* When this function is called with the following 'cmd's, the following
|
|
|
|
* is true while this function is being run:
|
|
|
|
* THREAD_NOFTIFY_SWTICH:
|
|
|
|
* - the previously running thread will not be scheduled onto another CPU.
|
|
|
|
* - the next thread to be run (v) will not be running on another CPU.
|
|
|
|
* - thread->cpu is the local CPU number
|
|
|
|
* - not preemptible as we're called in the middle of a thread switch
|
|
|
|
* THREAD_NOTIFY_FLUSH:
|
|
|
|
* - the thread (v) will be running on the local CPU, so
|
|
|
|
* v === current_thread_info()
|
|
|
|
* - thread->cpu is the local CPU number at the time it is accessed,
|
|
|
|
* but may change at any time.
|
|
|
|
* - we could be preempted if tree preempt rcu is enabled, so
|
|
|
|
* it is unsafe to use thread->cpu.
|
2009-12-18 15:34:43 +01:00
|
|
|
* THREAD_NOTIFY_EXIT
|
|
|
|
* - we could be preempted if tree preempt rcu is enabled, so
|
|
|
|
* it is unsafe to use thread->cpu.
|
2009-12-12 15:47:40 +01:00
|
|
|
*/
|
2006-06-21 14:31:52 +02:00
|
|
|
static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2006-06-21 14:31:52 +02:00
|
|
|
struct thread_info *thread = v;
|
2011-04-06 17:16:29 +02:00
|
|
|
u32 fpexc;
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
unsigned int cpu;
|
|
|
|
#endif
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2011-04-06 17:16:29 +02:00
|
|
|
switch (cmd) {
|
|
|
|
case THREAD_NOTIFY_SWITCH:
|
|
|
|
fpexc = fmrx(FPEXC);
|
2007-01-24 18:47:08 +01:00
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
2011-04-06 17:16:29 +02:00
|
|
|
cpu = thread->cpu;
|
2009-12-12 15:47:40 +01:00
|
|
|
|
2007-01-24 18:47:08 +01:00
|
|
|
/*
|
|
|
|
* On SMP, if VFP is enabled, save the old state in
|
|
|
|
* case the thread migrates to a different CPU. The
|
|
|
|
* restoring is done lazily.
|
|
|
|
*/
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu])
|
2011-07-09 14:44:04 +02:00
|
|
|
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
|
2007-01-24 18:47:08 +01:00
|
|
|
#endif
|
|
|
|
|
2006-08-27 13:38:34 +02:00
|
|
|
/*
|
|
|
|
* Always disable VFP so we can lazily save/restore the
|
|
|
|
* old state.
|
|
|
|
*/
|
2007-07-18 10:37:10 +02:00
|
|
|
fmxr(FPEXC, fpexc & ~FPEXC_EN);
|
2011-04-06 17:16:29 +02:00
|
|
|
break;
|
2006-08-27 13:38:34 +02:00
|
|
|
|
2011-04-06 17:16:29 +02:00
|
|
|
case THREAD_NOTIFY_FLUSH:
|
2009-12-12 15:47:40 +01:00
|
|
|
vfp_thread_flush(thread);
|
2011-04-06 17:16:29 +02:00
|
|
|
break;
|
|
|
|
|
|
|
|
case THREAD_NOTIFY_EXIT:
|
2009-12-18 15:34:43 +01:00
|
|
|
vfp_thread_exit(thread);
|
2011-04-06 17:17:17 +02:00
|
|
|
break;
|
|
|
|
|
|
|
|
case THREAD_NOTIFY_COPY:
|
|
|
|
vfp_thread_copy(thread);
|
2011-04-06 17:16:29 +02:00
|
|
|
break;
|
|
|
|
}
|
2006-08-27 13:38:34 +02:00
|
|
|
|
2006-06-21 14:31:52 +02:00
|
|
|
return NOTIFY_DONE;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2006-06-21 14:31:52 +02:00
|
|
|
static struct notifier_block vfp_notifier_block = {
|
|
|
|
.notifier_call = vfp_notifier,
|
|
|
|
};
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* Raise a SIGFPE for the current process.
|
|
|
|
* sicode describes the signal being raised.
|
|
|
|
*/
|
2011-01-08 13:05:09 +01:00
|
|
|
static void vfp_raise_sigfpe(unsigned int sicode, struct pt_regs *regs)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
siginfo_t info;
|
|
|
|
|
|
|
|
memset(&info, 0, sizeof(info));
|
|
|
|
|
|
|
|
info.si_signo = SIGFPE;
|
|
|
|
info.si_code = sicode;
|
2006-10-11 18:22:44 +02:00
|
|
|
info.si_addr = (void __user *)(instruction_pointer(regs) - 4);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This is the same as NWFPE, because it's not clear what
|
|
|
|
* this is used for
|
|
|
|
*/
|
|
|
|
current->thread.error_code = 0;
|
|
|
|
current->thread.trap_no = 6;
|
|
|
|
|
2005-06-30 00:02:02 +02:00
|
|
|
send_sig_info(SIGFPE, &info, current);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2007-11-22 18:32:01 +01:00
|
|
|
static void vfp_panic(char *reason, u32 inst)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2012-05-09 23:32:43 +02:00
|
|
|
pr_err("VFP: Error: %s\n", reason);
|
|
|
|
pr_err("VFP: EXC 0x%08x SCR 0x%08x INST 0x%08x\n",
|
2007-11-22 18:32:01 +01:00
|
|
|
fmrx(FPEXC), fmrx(FPSCR), inst);
|
2005-04-17 00:20:36 +02:00
|
|
|
for (i = 0; i < 32; i += 2)
|
2012-05-09 23:32:43 +02:00
|
|
|
pr_err("VFP: s%2u: 0x%08x s%2u: 0x%08x\n",
|
2005-04-17 00:20:36 +02:00
|
|
|
i, vfp_get_float(i), i+1, vfp_get_float(i+1));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Process bitmask of exception conditions.
|
|
|
|
*/
|
|
|
|
static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
int si_code = 0;
|
|
|
|
|
|
|
|
pr_debug("VFP: raising exceptions %08x\n", exceptions);
|
|
|
|
|
2006-08-27 13:42:08 +02:00
|
|
|
if (exceptions == VFP_EXCEPTION_ERROR) {
|
2007-11-22 18:32:01 +01:00
|
|
|
vfp_panic("unhandled bounce", inst);
|
2005-04-17 00:20:36 +02:00
|
|
|
vfp_raise_sigfpe(0, regs);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2010-02-01 18:50:40 +01:00
|
|
|
* If any of the status flags are set, update the FPSCR.
|
2005-04-17 00:20:36 +02:00
|
|
|
* Comparison instructions always return at least one of
|
|
|
|
* these flags set.
|
|
|
|
*/
|
2010-02-01 18:50:40 +01:00
|
|
|
if (exceptions & (FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V))
|
|
|
|
fpscr &= ~(FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V);
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
fpscr |= exceptions;
|
|
|
|
|
|
|
|
fmxr(FPSCR, fpscr);
|
|
|
|
|
|
|
|
#define RAISE(stat,en,sig) \
|
|
|
|
if (exceptions & stat && fpscr & en) \
|
|
|
|
si_code = sig;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* These are arranged in priority order, least to highest.
|
|
|
|
*/
|
2006-10-23 12:19:40 +02:00
|
|
|
RAISE(FPSCR_DZC, FPSCR_DZE, FPE_FLTDIV);
|
2005-04-17 00:20:36 +02:00
|
|
|
RAISE(FPSCR_IXC, FPSCR_IXE, FPE_FLTRES);
|
|
|
|
RAISE(FPSCR_UFC, FPSCR_UFE, FPE_FLTUND);
|
|
|
|
RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);
|
|
|
|
RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);
|
|
|
|
|
|
|
|
if (si_code)
|
|
|
|
vfp_raise_sigfpe(si_code, regs);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Emulate a VFP instruction.
|
|
|
|
*/
|
|
|
|
static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs)
|
|
|
|
{
|
2006-08-27 13:42:08 +02:00
|
|
|
u32 exceptions = VFP_EXCEPTION_ERROR;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
pr_debug("VFP: emulate: INST=0x%08x SCR=0x%08x\n", inst, fpscr);
|
|
|
|
|
|
|
|
if (INST_CPRTDO(inst)) {
|
|
|
|
if (!INST_CPRT(inst)) {
|
|
|
|
/*
|
|
|
|
* CPDO
|
|
|
|
*/
|
|
|
|
if (vfp_single(inst)) {
|
|
|
|
exceptions = vfp_single_cpdo(inst, fpscr);
|
|
|
|
} else {
|
|
|
|
exceptions = vfp_double_cpdo(inst, fpscr);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* A CPRT instruction can not appear in FPINST2, nor
|
|
|
|
* can it cause an exception. Therefore, we do not
|
|
|
|
* have to emulate it.
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* A CPDT instruction can not appear in FPINST2, nor can
|
|
|
|
* it cause an exception. Therefore, we do not have to
|
|
|
|
* emulate it.
|
|
|
|
*/
|
|
|
|
}
|
2006-04-25 21:41:27 +02:00
|
|
|
return exceptions & ~VFP_NAN_FLAG;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Package up a bounce condition.
|
|
|
|
*/
|
2007-11-22 18:32:01 +01:00
|
|
|
void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2007-11-22 18:32:01 +01:00
|
|
|
u32 fpscr, orig_fpscr, fpsid, exceptions;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
|
|
|
|
|
|
|
|
/*
|
2007-11-22 18:32:01 +01:00
|
|
|
* At this point, FPEXC can have the following configuration:
|
|
|
|
*
|
|
|
|
* EX DEX IXE
|
|
|
|
* 0 1 x - synchronous exception
|
|
|
|
* 1 x 0 - asynchronous exception
|
|
|
|
* 1 x 1 - sychronous on VFP subarch 1 and asynchronous on later
|
|
|
|
* 0 0 1 - synchronous on VFP9 (non-standard subarch 1
|
|
|
|
* implementation), undefined otherwise
|
|
|
|
*
|
|
|
|
* Clear various bits and enable access to the VFP so we can
|
|
|
|
* handle the bounce.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
2007-11-22 18:32:01 +01:00
|
|
|
fmxr(FPEXC, fpexc & ~(FPEXC_EX|FPEXC_DEX|FPEXC_FP2V|FPEXC_VV|FPEXC_TRAP_MASK));
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-11-22 18:32:01 +01:00
|
|
|
fpsid = fmrx(FPSID);
|
2005-04-17 00:20:36 +02:00
|
|
|
orig_fpscr = fpscr = fmrx(FPSCR);
|
|
|
|
|
|
|
|
/*
|
2007-11-22 18:32:01 +01:00
|
|
|
* Check for the special VFP subarch 1 and FPSCR.IXE bit case
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
2007-11-22 18:32:01 +01:00
|
|
|
if ((fpsid & FPSID_ARCH_MASK) == (1 << FPSID_ARCH_BIT)
|
|
|
|
&& (fpscr & FPSCR_IXE)) {
|
|
|
|
/*
|
|
|
|
* Synchronous exception, emulate the trigger instruction
|
|
|
|
*/
|
2005-04-17 00:20:36 +02:00
|
|
|
goto emulate;
|
|
|
|
}
|
|
|
|
|
2007-11-22 18:32:01 +01:00
|
|
|
if (fpexc & FPEXC_EX) {
|
2009-05-30 15:00:18 +02:00
|
|
|
#ifndef CONFIG_CPU_FEROCEON
|
2007-11-22 18:32:01 +01:00
|
|
|
/*
|
|
|
|
* Asynchronous exception. The instruction is read from FPINST
|
|
|
|
* and the interrupted instruction has to be restarted.
|
|
|
|
*/
|
|
|
|
trigger = fmrx(FPINST);
|
|
|
|
regs->ARM_pc -= 4;
|
2009-05-30 15:00:18 +02:00
|
|
|
#endif
|
2007-11-22 18:32:01 +01:00
|
|
|
} else if (!(fpexc & FPEXC_DEX)) {
|
|
|
|
/*
|
|
|
|
* Illegal combination of bits. It can be caused by an
|
|
|
|
* unallocated VFP instruction but with FPSCR.IXE set and not
|
|
|
|
* on VFP subarch 1.
|
|
|
|
*/
|
|
|
|
vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs);
|
2009-04-01 21:27:18 +02:00
|
|
|
goto exit;
|
2007-11-22 18:32:01 +01:00
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
/*
|
2007-11-22 18:32:01 +01:00
|
|
|
* Modify fpscr to indicate the number of iterations remaining.
|
|
|
|
* If FPEXC.EX is 0, FPEXC.DEX is 1 and the FPEXC.VV bit indicates
|
|
|
|
* whether FPEXC.VECITR or FPSCR.LEN is used.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
2007-11-22 18:32:01 +01:00
|
|
|
if (fpexc & (FPEXC_EX | FPEXC_VV)) {
|
2005-04-17 00:20:36 +02:00
|
|
|
u32 len;
|
|
|
|
|
|
|
|
len = fpexc + (1 << FPEXC_LENGTH_BIT);
|
|
|
|
|
|
|
|
fpscr &= ~FPSCR_LENGTH_MASK;
|
|
|
|
fpscr |= (len & FPEXC_LENGTH_MASK) << (FPSCR_LENGTH_BIT - FPEXC_LENGTH_BIT);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Handle the first FP instruction. We used to take note of the
|
|
|
|
* FPEXC bounce reason, but this appears to be unreliable.
|
|
|
|
* Emulate the bounced instruction instead.
|
|
|
|
*/
|
2007-11-22 18:32:01 +01:00
|
|
|
exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (exceptions)
|
2007-11-22 18:32:01 +01:00
|
|
|
vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
/*
|
2007-11-22 18:32:01 +01:00
|
|
|
* If there isn't a second FP instruction, exit now. Note that
|
|
|
|
* the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
ARM: VFP: fix emulation of second VFP instruction
Martin Storsjö reports that the sequence:
ee312ac1 vsub.f32 s4, s3, s2
ee702ac0 vsub.f32 s5, s1, s0
e59f0028 ldr r0, [pc, #40]
ee111a90 vmov r1, s3
on Raspberry Pi (implementor 41 architecture 1 part 20 variant b rev 5)
where s3 is a denormal and s2 is zero results in incorrect behaviour -
the instruction "vsub.f32 s5, s1, s0" is not executed:
VFP: bounce: trigger ee111a90 fpexc d0000780
VFP: emulate: INST=0xee312ac1 SCR=0x00000000
...
As we can see, the instruction triggering the exception is the "vmov"
instruction, and we emulate the "vsub.f32 s4, s3, s2" but fail to
properly take account of the FPEXC_FP2V flag in FPEXC. This is because
the test for the second instruction register being valid is bogus, and
will always skip emulation of the second instruction.
Cc: <stable@vger.kernel.org>
Reported-by: Martin Storsjö <martin@martin.st>
Tested-by: Martin Storsjö <martin@martin.st>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2013-02-25 17:09:12 +01:00
|
|
|
if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V))
|
2009-04-01 21:27:18 +02:00
|
|
|
goto exit;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The barrier() here prevents fpinst2 being read
|
|
|
|
* before the condition above.
|
|
|
|
*/
|
|
|
|
barrier();
|
|
|
|
trigger = fmrx(FPINST2);
|
|
|
|
|
|
|
|
emulate:
|
2007-11-22 18:32:01 +01:00
|
|
|
exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (exceptions)
|
|
|
|
vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
|
2009-04-01 21:27:18 +02:00
|
|
|
exit:
|
|
|
|
preempt_enable();
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2006-12-08 16:22:20 +01:00
|
|
|
|
2007-01-03 00:40:30 +01:00
|
|
|
static void vfp_enable(void *unused)
|
|
|
|
{
|
2012-05-11 18:42:37 +02:00
|
|
|
u32 access;
|
|
|
|
|
|
|
|
BUG_ON(preemptible());
|
|
|
|
access = get_copro_access();
|
2007-01-03 00:40:30 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Enable full access to VFP (cp10 and cp11)
|
|
|
|
*/
|
|
|
|
set_copro_access(access | CPACC_FULL(10) | CPACC_FULL(11));
|
|
|
|
}
|
|
|
|
|
2015-04-18 01:53:58 +02:00
|
|
|
/* Called by platforms on which we want to disable VFP because it may not be
|
|
|
|
* present on all CPUs within a SMP complex. Needs to be called prior to
|
|
|
|
* vfp_init().
|
|
|
|
*/
|
|
|
|
void vfp_disable(void)
|
|
|
|
{
|
|
|
|
if (VFP_arch) {
|
|
|
|
pr_debug("%s: should be called prior to vfp_init\n", __func__);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
VFP_arch = 1;
|
|
|
|
}
|
|
|
|
|
2011-02-10 11:08:32 +01:00
|
|
|
#ifdef CONFIG_CPU_PM
|
2011-04-22 22:02:33 +02:00
|
|
|
static int vfp_pm_suspend(void)
|
2008-12-18 12:26:54 +01:00
|
|
|
{
|
|
|
|
struct thread_info *ti = current_thread_info();
|
|
|
|
u32 fpexc = fmrx(FPEXC);
|
|
|
|
|
|
|
|
/* if vfp is on, then save state for resumption */
|
|
|
|
if (fpexc & FPEXC_EN) {
|
2012-05-09 23:32:43 +02:00
|
|
|
pr_debug("%s: saving vfp state\n", __func__);
|
2008-12-18 12:26:54 +01:00
|
|
|
vfp_save_state(&ti->vfpstate, fpexc);
|
|
|
|
|
|
|
|
/* disable, just in case */
|
|
|
|
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
|
2012-07-20 03:03:42 +02:00
|
|
|
} else if (vfp_current_hw_state[ti->cpu]) {
|
|
|
|
#ifndef CONFIG_SMP
|
|
|
|
fmxr(FPEXC, fpexc | FPEXC_EN);
|
|
|
|
vfp_save_state(vfp_current_hw_state[ti->cpu], fpexc);
|
|
|
|
fmxr(FPEXC, fpexc);
|
|
|
|
#endif
|
2008-12-18 12:26:54 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* clear any information we had about last context state */
|
2012-07-20 03:03:43 +02:00
|
|
|
vfp_current_hw_state[ti->cpu] = NULL;
|
2008-12-18 12:26:54 +01:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-04-22 22:02:33 +02:00
|
|
|
static void vfp_pm_resume(void)
|
2008-12-18 12:26:54 +01:00
|
|
|
{
|
|
|
|
/* ensure we have access to the vfp */
|
|
|
|
vfp_enable(NULL);
|
|
|
|
|
|
|
|
/* and disable it to ensure the next usage restores the state */
|
|
|
|
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
|
|
|
|
}
|
|
|
|
|
2011-02-10 11:08:32 +01:00
|
|
|
static int vfp_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd,
|
|
|
|
void *v)
|
|
|
|
{
|
|
|
|
switch (cmd) {
|
|
|
|
case CPU_PM_ENTER:
|
|
|
|
vfp_pm_suspend();
|
|
|
|
break;
|
|
|
|
case CPU_PM_ENTER_FAILED:
|
|
|
|
case CPU_PM_EXIT:
|
|
|
|
vfp_pm_resume();
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return NOTIFY_OK;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct notifier_block vfp_cpu_pm_notifier_block = {
|
|
|
|
.notifier_call = vfp_cpu_pm_notifier,
|
2008-12-18 12:26:54 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
static void vfp_pm_init(void)
|
|
|
|
{
|
2011-02-10 11:08:32 +01:00
|
|
|
cpu_pm_register_notifier(&vfp_cpu_pm_notifier_block);
|
2008-12-18 12:26:54 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
static inline void vfp_pm_init(void) { }
|
2011-02-10 11:08:32 +01:00
|
|
|
#endif /* CONFIG_CPU_PM */
|
2008-12-18 12:26:54 +01:00
|
|
|
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
/*
|
|
|
|
* Ensure that the VFP state stored in 'thread->vfpstate' is up to date
|
|
|
|
* with the hardware state.
|
|
|
|
*/
|
2010-02-06 12:36:23 +01:00
|
|
|
void vfp_sync_hwstate(struct thread_info *thread)
|
2009-02-11 13:12:56 +01:00
|
|
|
{
|
|
|
|
unsigned int cpu = get_cpu();
|
|
|
|
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
if (vfp_state_in_hw(cpu, thread)) {
|
2010-02-06 12:27:45 +01:00
|
|
|
u32 fpexc = fmrx(FPEXC);
|
2009-02-11 13:12:56 +01:00
|
|
|
|
2010-02-06 12:27:45 +01:00
|
|
|
/*
|
|
|
|
* Save the last VFP state on this CPU.
|
|
|
|
*/
|
|
|
|
fmxr(FPEXC, fpexc | FPEXC_EN);
|
|
|
|
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
|
2010-02-06 12:36:23 +01:00
|
|
|
fmxr(FPEXC, fpexc);
|
|
|
|
}
|
2009-02-11 13:12:56 +01:00
|
|
|
|
2010-02-06 12:36:23 +01:00
|
|
|
put_cpu();
|
|
|
|
}
|
|
|
|
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
/* Ensure that the thread reloads the hardware VFP state on the next use. */
|
2010-02-06 12:36:23 +01:00
|
|
|
void vfp_flush_hwstate(struct thread_info *thread)
|
|
|
|
{
|
|
|
|
unsigned int cpu = get_cpu();
|
2009-02-11 13:12:56 +01:00
|
|
|
|
ARM: vfp: fix a hole in VFP thread migration
Fix a hole in the VFP thread migration. Lets define two threads.
Thread 1, we'll call 'interesting_thread' which is a thread which is
running on CPU0, using VFP (so vfp_current_hw_state[0] =
&interesting_thread->vfpstate) and gets migrated off to CPU1, where
it continues execution of VFP instructions.
Thread 2, we'll call 'new_cpu0_thread' which is the thread which takes
over on CPU0. This has also been using VFP, and last used VFP on CPU0,
but doesn't use it again.
The following code will be executed twice:
cpu = thread->cpu;
/*
* On SMP, if VFP is enabled, save the old state in
* case the thread migrates to a different CPU. The
* restoring is done lazily.
*/
if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) {
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
vfp_current_hw_state[cpu]->hard.cpu = cpu;
}
/*
* Thread migration, just force the reloading of the
* state on the new CPU in case the VFP registers
* contain stale data.
*/
if (thread->vfpstate.hard.cpu != cpu)
vfp_current_hw_state[cpu] = NULL;
The first execution will be on CPU0 to switch away from 'interesting_thread'.
interesting_thread->cpu will be 0.
So, vfp_current_hw_state[0] points at interesting_thread->vfpstate.
The hardware state will be saved, along with the CPU number (0) that
it was executing on.
'thread' will be 'new_cpu0_thread' with new_cpu0_thread->cpu = 0.
Also, because it was executing on CPU0, new_cpu0_thread->vfpstate.hard.cpu = 0,
and so the thread migration check is not triggered.
This means that vfp_current_hw_state[0] remains pointing at interesting_thread.
The second execution will be on CPU1 to switch _to_ 'interesting_thread'.
So, 'thread' will be 'interesting_thread' and interesting_thread->cpu now
will be 1. The previous thread executing on CPU1 is not relevant to this
so we shall ignore that.
We get to the thread migration check. Here, we discover that
interesting_thread->vfpstate.hard.cpu = 0, yet interesting_thread->cpu is
now 1, indicating thread migration. We set vfp_current_hw_state[1] to
NULL.
So, at this point vfp_current_hw_state[] contains the following:
[0] = &interesting_thread->vfpstate
[1] = NULL
Our interesting thread now executes a VFP instruction, takes a fault
which loads the state into the VFP hardware. Now, through the assembly
we now have:
[0] = &interesting_thread->vfpstate
[1] = &interesting_thread->vfpstate
CPU1 stops due to ptrace (and so saves its VFP state) using the thread
switch code above), and CPU0 calls vfp_sync_hwstate().
if (vfp_current_hw_state[cpu] == &thread->vfpstate) {
vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
BANG, we corrupt interesting_thread's VFP state by overwriting the
more up-to-date state saved by CPU1 with the old VFP state from CPU0.
Fix this by ensuring that we have sane semantics for the various state
describing variables:
1. vfp_current_hw_state[] points to the current owner of the context
information stored in each CPUs hardware, or NULL if that state
information is invalid.
2. thread->vfpstate.hard.cpu always contains the most recent CPU number
which the state was loaded into or NR_CPUS if no CPU owns the state.
So, for a particular CPU to be a valid owner of the VFP state for a
particular thread t, two things must be true:
vfp_current_hw_state[cpu] == &t->vfpstate && t->vfpstate.hard.cpu == cpu.
and that is valid from the moment a CPU loads the saved VFP context
into the hardware. This gives clear and consistent semantics to
interpreting these variables.
This patch also fixes thread copying, ensuring that t->vfpstate.hard.cpu
is invalidated, otherwise CPU0 may believe it was the last owner. The
hole can happen thus:
- thread1 runs on CPU2 using VFP, migrates to CPU3, exits and thread_info
freed.
- New thread allocated from a previously running thread on CPU2, reusing
memory for thread1 and copying vfp.hard.cpu.
At this point, the following are true:
new_thread1->vfpstate.hard.cpu == 2
&new_thread1->vfpstate == vfp_current_hw_state[2]
Lastly, this also addresses thread flushing in a similar way to thread
copying. Hole is:
- thread runs on CPU0, using VFP, migrates to CPU1 but does not use VFP.
- thread calls execve(), so thread flush happens, leaving
vfp_current_hw_state[0] intact. This vfpstate is memset to 0 causing
thread->vfpstate.hard.cpu = 0.
- thread migrates back to CPU0 before using VFP.
At this point, the following are true:
thread->vfpstate.hard.cpu == 0
&thread->vfpstate == vfp_current_hw_state[0]
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2011-07-09 17:09:43 +02:00
|
|
|
vfp_force_reload(cpu, thread);
|
2010-02-06 12:36:23 +01:00
|
|
|
|
2009-02-11 13:12:56 +01:00
|
|
|
put_cpu();
|
|
|
|
}
|
|
|
|
|
2012-04-23 16:38:28 +02:00
|
|
|
/*
|
|
|
|
* Save the current VFP state into the provided structures and prepare
|
|
|
|
* for entry into a new function (signal handler).
|
|
|
|
*/
|
|
|
|
int vfp_preserve_user_clear_hwstate(struct user_vfp __user *ufp,
|
|
|
|
struct user_vfp_exc __user *ufp_exc)
|
|
|
|
{
|
|
|
|
struct thread_info *thread = current_thread_info();
|
|
|
|
struct vfp_hard_struct *hwstate = &thread->vfpstate.hard;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
/* Ensure that the saved hwstate is up-to-date. */
|
|
|
|
vfp_sync_hwstate(thread);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Copy the floating point registers. There can be unused
|
|
|
|
* registers see asm/hwcap.h for details.
|
|
|
|
*/
|
|
|
|
err |= __copy_to_user(&ufp->fpregs, &hwstate->fpregs,
|
|
|
|
sizeof(hwstate->fpregs));
|
|
|
|
/*
|
|
|
|
* Copy the status and control register.
|
|
|
|
*/
|
|
|
|
__put_user_error(hwstate->fpscr, &ufp->fpscr, err);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Copy the exception registers.
|
|
|
|
*/
|
|
|
|
__put_user_error(hwstate->fpexc, &ufp_exc->fpexc, err);
|
|
|
|
__put_user_error(hwstate->fpinst, &ufp_exc->fpinst, err);
|
|
|
|
__put_user_error(hwstate->fpinst2, &ufp_exc->fpinst2, err);
|
|
|
|
|
|
|
|
if (err)
|
|
|
|
return -EFAULT;
|
2012-04-23 16:42:16 +02:00
|
|
|
|
|
|
|
/* Ensure that VFP is disabled. */
|
|
|
|
vfp_flush_hwstate(thread);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* As per the PCS, clear the length and stride bits for function
|
|
|
|
* entry.
|
|
|
|
*/
|
|
|
|
hwstate->fpscr &= ~(FPSCR_LENGTH_MASK | FPSCR_STRIDE_MASK);
|
2012-04-23 16:38:28 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Sanitise and restore the current VFP state from the provided structures. */
|
|
|
|
int vfp_restore_user_hwstate(struct user_vfp __user *ufp,
|
|
|
|
struct user_vfp_exc __user *ufp_exc)
|
|
|
|
{
|
|
|
|
struct thread_info *thread = current_thread_info();
|
|
|
|
struct vfp_hard_struct *hwstate = &thread->vfpstate.hard;
|
|
|
|
unsigned long fpexc;
|
|
|
|
int err = 0;
|
|
|
|
|
2012-05-15 16:51:54 +02:00
|
|
|
/* Disable VFP to avoid corrupting the new thread state. */
|
|
|
|
vfp_flush_hwstate(thread);
|
2012-04-23 16:38:28 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Copy the floating point registers. There can be unused
|
|
|
|
* registers see asm/hwcap.h for details.
|
|
|
|
*/
|
|
|
|
err |= __copy_from_user(&hwstate->fpregs, &ufp->fpregs,
|
|
|
|
sizeof(hwstate->fpregs));
|
|
|
|
/*
|
|
|
|
* Copy the status and control register.
|
|
|
|
*/
|
|
|
|
__get_user_error(hwstate->fpscr, &ufp->fpscr, err);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Sanitise and restore the exception registers.
|
|
|
|
*/
|
|
|
|
__get_user_error(fpexc, &ufp_exc->fpexc, err);
|
|
|
|
|
|
|
|
/* Ensure the VFP is enabled. */
|
|
|
|
fpexc |= FPEXC_EN;
|
|
|
|
|
|
|
|
/* Ensure FPINST2 is invalid and the exception flag is cleared. */
|
|
|
|
fpexc &= ~(FPEXC_EX | FPEXC_FP2V);
|
|
|
|
hwstate->fpexc = fpexc;
|
|
|
|
|
|
|
|
__get_user_error(hwstate->fpinst, &ufp_exc->fpinst, err);
|
|
|
|
__get_user_error(hwstate->fpinst2, &ufp_exc->fpinst2, err);
|
|
|
|
|
|
|
|
return err ? -EFAULT : 0;
|
|
|
|
}
|
|
|
|
|
2010-12-18 11:59:49 +01:00
|
|
|
/*
|
|
|
|
* VFP hardware can lose all context when a CPU goes offline.
|
2011-01-31 15:43:25 +01:00
|
|
|
* As we will be running in SMP mode with CPU hotplug, we will save the
|
|
|
|
* hardware state at every thread switch. We clear our held state when
|
|
|
|
* a CPU has been killed, indicating that the VFP hardware doesn't contain
|
|
|
|
* a threads VFP state. When a CPU starts up, we re-enable access to the
|
2016-07-13 19:16:35 +02:00
|
|
|
* VFP hardware. The callbacks below are called on the CPU which
|
2010-12-18 11:59:49 +01:00
|
|
|
* is being offlined/onlined.
|
|
|
|
*/
|
2016-07-13 19:16:35 +02:00
|
|
|
static int vfp_dying_cpu(unsigned int cpu)
|
2010-12-18 11:59:49 +01:00
|
|
|
{
|
2016-07-13 19:16:35 +02:00
|
|
|
vfp_force_reload(cpu, current_thread_info());
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int vfp_starting_cpu(unsigned int unused)
|
|
|
|
{
|
|
|
|
vfp_enable(NULL);
|
|
|
|
return 0;
|
2010-12-18 11:59:49 +01:00
|
|
|
}
|
2007-01-03 00:40:30 +01:00
|
|
|
|
2013-05-24 16:23:28 +02:00
|
|
|
void vfp_kmode_exception(void)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If we reach this point, a floating point exception has been raised
|
|
|
|
* while running in kernel mode. If the NEON/VFP unit was enabled at the
|
|
|
|
* time, it means a VFP instruction has been issued that requires
|
|
|
|
* software assistance to complete, something which is not currently
|
|
|
|
* supported in kernel mode.
|
|
|
|
* If the NEON/VFP unit was disabled, and the location pointed to below
|
|
|
|
* is properly preceded by a call to kernel_neon_begin(), something has
|
|
|
|
* caused the task to be scheduled out and back in again. In this case,
|
|
|
|
* rebuilding and running with CONFIG_DEBUG_ATOMIC_SLEEP enabled should
|
|
|
|
* be helpful in localizing the problem.
|
|
|
|
*/
|
|
|
|
if (fmrx(FPEXC) & FPEXC_EN)
|
|
|
|
pr_crit("BUG: unsupported FP instruction in kernel mode\n");
|
|
|
|
else
|
|
|
|
pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n");
|
|
|
|
}
|
|
|
|
|
2013-05-16 11:41:48 +02:00
|
|
|
#ifdef CONFIG_KERNEL_MODE_NEON
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Kernel-side NEON support functions
|
|
|
|
*/
|
|
|
|
void kernel_neon_begin(void)
|
|
|
|
{
|
|
|
|
struct thread_info *thread = current_thread_info();
|
|
|
|
unsigned int cpu;
|
|
|
|
u32 fpexc;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Kernel mode NEON is only allowed outside of interrupt context
|
|
|
|
* with preemption disabled. This will make sure that the kernel
|
|
|
|
* mode NEON register contents never need to be preserved.
|
|
|
|
*/
|
|
|
|
BUG_ON(in_interrupt());
|
|
|
|
cpu = get_cpu();
|
|
|
|
|
|
|
|
fpexc = fmrx(FPEXC) | FPEXC_EN;
|
|
|
|
fmxr(FPEXC, fpexc);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Save the userland NEON/VFP state. Under UP,
|
|
|
|
* the owner could be a task other than 'current'
|
|
|
|
*/
|
|
|
|
if (vfp_state_in_hw(cpu, thread))
|
|
|
|
vfp_save_state(&thread->vfpstate, fpexc);
|
|
|
|
#ifndef CONFIG_SMP
|
|
|
|
else if (vfp_current_hw_state[cpu] != NULL)
|
|
|
|
vfp_save_state(vfp_current_hw_state[cpu], fpexc);
|
|
|
|
#endif
|
|
|
|
vfp_current_hw_state[cpu] = NULL;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kernel_neon_begin);
|
|
|
|
|
|
|
|
void kernel_neon_end(void)
|
|
|
|
{
|
|
|
|
/* Disable the NEON/VFP unit. */
|
|
|
|
fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
|
|
|
|
put_cpu();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kernel_neon_end);
|
|
|
|
|
|
|
|
#endif /* CONFIG_KERNEL_MODE_NEON */
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* VFP support code initialisation.
|
|
|
|
*/
|
|
|
|
static int __init vfp_init(void)
|
|
|
|
{
|
|
|
|
unsigned int vfpsid;
|
2006-12-08 16:22:20 +01:00
|
|
|
unsigned int cpu_arch = cpu_architecture();
|
|
|
|
|
2016-07-13 19:16:35 +02:00
|
|
|
/*
|
|
|
|
* Enable the access to the VFP on all online CPUs so the
|
|
|
|
* following test on FPSID will succeed.
|
|
|
|
*/
|
2007-11-22 18:32:01 +01:00
|
|
|
if (cpu_arch >= CPU_ARCH_ARMv6)
|
2012-05-11 18:42:37 +02:00
|
|
|
on_each_cpu(vfp_enable, NULL, 1);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* First check that there is a VFP that we can use.
|
|
|
|
* The handler is already setup to just log calls, so
|
|
|
|
* we just need to read the VFPSID register.
|
|
|
|
*/
|
2007-06-10 13:22:20 +02:00
|
|
|
vfp_vector = vfp_testing_entry;
|
2007-09-09 15:24:59 +02:00
|
|
|
barrier();
|
2005-04-17 00:20:36 +02:00
|
|
|
vfpsid = fmrx(FPSID);
|
2007-01-03 00:40:30 +01:00
|
|
|
barrier();
|
2007-06-10 13:22:20 +02:00
|
|
|
vfp_vector = vfp_null_entry;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2012-05-09 23:32:43 +02:00
|
|
|
pr_info("VFP support v0.3: ");
|
2014-11-10 21:56:41 +01:00
|
|
|
if (VFP_arch) {
|
2012-05-09 23:32:43 +02:00
|
|
|
pr_cont("not present\n");
|
2014-11-10 21:56:41 +01:00
|
|
|
return 0;
|
|
|
|
/* Extract the architecture on CPUID scheme */
|
|
|
|
} else if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) {
|
|
|
|
VFP_arch = vfpsid & FPSID_CPUID_ARCH_MASK;
|
|
|
|
VFP_arch >>= FPSID_ARCH_BIT;
|
2006-12-08 16:22:20 +01:00
|
|
|
/*
|
2014-11-10 21:56:41 +01:00
|
|
|
* Check for the presence of the Advanced SIMD
|
|
|
|
* load/store instructions, integer and single
|
|
|
|
* precision floating point operations. Only check
|
|
|
|
* for NEON if the hardware has the MVFR registers.
|
2006-12-08 16:22:20 +01:00
|
|
|
*/
|
2014-11-20 20:29:09 +01:00
|
|
|
if (IS_ENABLED(CONFIG_NEON) &&
|
|
|
|
(fmrx(MVFR1) & 0x000fff00) == 0x00011100)
|
2014-11-10 21:56:41 +01:00
|
|
|
elf_hwcap |= HWCAP_NEON;
|
|
|
|
|
2014-11-20 20:29:09 +01:00
|
|
|
if (IS_ENABLED(CONFIG_VFPv3)) {
|
|
|
|
u32 mvfr0 = fmrx(MVFR0);
|
|
|
|
if (((mvfr0 & MVFR0_DP_MASK) >> MVFR0_DP_BIT) == 0x2 ||
|
|
|
|
((mvfr0 & MVFR0_SP_MASK) >> MVFR0_SP_BIT) == 0x2) {
|
|
|
|
elf_hwcap |= HWCAP_VFPv3;
|
|
|
|
/*
|
|
|
|
* Check for VFPv3 D16 and VFPv4 D16. CPUs in
|
|
|
|
* this configuration only have 16 x 64bit
|
|
|
|
* registers.
|
|
|
|
*/
|
|
|
|
if ((mvfr0 & MVFR0_A_SIMD_MASK) == 1)
|
|
|
|
/* also v4-D16 */
|
|
|
|
elf_hwcap |= HWCAP_VFPv3D16;
|
|
|
|
else
|
|
|
|
elf_hwcap |= HWCAP_VFPD32;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((fmrx(MVFR1) & 0xf0000000) == 0x10000000)
|
|
|
|
elf_hwcap |= HWCAP_VFPv4;
|
|
|
|
}
|
2014-11-10 21:56:41 +01:00
|
|
|
/* Extract the architecture version on pre-cpuid scheme */
|
|
|
|
} else {
|
|
|
|
if (vfpsid & FPSID_NODOUBLE) {
|
|
|
|
pr_cont("no double precision support\n");
|
|
|
|
return 0;
|
2011-06-08 12:42:43 +02:00
|
|
|
}
|
2014-11-10 21:56:41 +01:00
|
|
|
|
|
|
|
VFP_arch = (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2014-11-10 21:56:41 +01:00
|
|
|
|
2016-07-13 19:16:35 +02:00
|
|
|
cpuhp_setup_state_nocalls(CPUHP_AP_ARM_VFP_STARTING,
|
|
|
|
"AP_ARM_VFP_STARTING", vfp_starting_cpu,
|
|
|
|
vfp_dying_cpu);
|
2014-11-10 21:56:41 +01:00
|
|
|
|
|
|
|
vfp_vector = vfp_support_entry;
|
|
|
|
|
|
|
|
thread_register_notifier(&vfp_notifier_block);
|
|
|
|
vfp_pm_init();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We detected VFP, and the support code is
|
|
|
|
* in place; report VFP support to userspace.
|
|
|
|
*/
|
|
|
|
elf_hwcap |= HWCAP_VFP;
|
|
|
|
|
|
|
|
pr_cont("implementor %02x architecture %d part %02x variant %x rev %x\n",
|
|
|
|
(vfpsid & FPSID_IMPLEMENTER_MASK) >> FPSID_IMPLEMENTER_BIT,
|
|
|
|
VFP_arch,
|
|
|
|
(vfpsid & FPSID_PART_MASK) >> FPSID_PART_BIT,
|
|
|
|
(vfpsid & FPSID_VARIANT_MASK) >> FPSID_VARIANT_BIT,
|
|
|
|
(vfpsid & FPSID_REV_MASK) >> FPSID_REV_BIT);
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-05-22 10:38:53 +02:00
|
|
|
core_initcall(vfp_init);
|