blob: 5f113bedfaee6d74c29628febf1b093ba6dffcf2 [file] [log] [blame]
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
Christoffer Dall342cd0a2013-01-20 18:28:06 -050018
19#include <linux/linkage.h>
20#include <linux/const.h>
21#include <asm/unified.h>
22#include <asm/page.h>
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050023#include <asm/ptrace.h>
Christoffer Dall749cf76c2013-01-20 18:28:06 -050024#include <asm/asm-offsets.h>
25#include <asm/kvm_asm.h>
Christoffer Dall342cd0a2013-01-20 18:28:06 -050026#include <asm/kvm_arm.h>
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050027#include <asm/vfpmacros.h>
28#include "interrupts_head.S"
Christoffer Dall342cd0a2013-01-20 18:28:06 -050029
30 .text
31
32__kvm_hyp_code_start:
33 .globl __kvm_hyp_code_start
34
35/********************************************************************
36 * Flush per-VMID TLBs
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050037 *
38 * void __kvm_tlb_flush_vmid(struct kvm *kvm);
39 *
40 * We rely on the hardware to broadcast the TLB invalidation to all CPUs
41 * inside the inner-shareable domain (which is the case for all v7
42 * implementations). If we come across a non-IS SMP implementation, we'll
43 * have to use an IPI based mechanism. Until then, we stick to the simple
44 * hardware assisted version.
Christoffer Dall342cd0a2013-01-20 18:28:06 -050045 */
Christoffer Dalld5d81842013-01-20 18:28:07 -050046ENTRY(__kvm_tlb_flush_vmid)
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050047 push {r2, r3}
48
49 add r0, r0, #KVM_VTTBR
50 ldrd r2, r3, [r0]
51 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
52 isb
53 mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored)
54 dsb
55 isb
56 mov r2, #0
57 mov r3, #0
58 mcrr p15, 6, r2, r3, c2 @ Back to VMID #0
59 isb @ Not necessary if followed by eret
60
61 pop {r2, r3}
Christoffer Dalld5d81842013-01-20 18:28:07 -050062 bx lr
63ENDPROC(__kvm_tlb_flush_vmid)
64
65/********************************************************************
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050066 * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
67 * domain, for all VMIDs
68 *
69 * void __kvm_flush_vm_context(void);
Christoffer Dalld5d81842013-01-20 18:28:07 -050070 */
Christoffer Dall342cd0a2013-01-20 18:28:06 -050071ENTRY(__kvm_flush_vm_context)
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050072 mov r0, #0 @ rn parameter for c15 flushes is SBZ
73
74 /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
75 mcr p15, 4, r0, c8, c3, 4
76 /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
77 mcr p15, 0, r0, c7, c1, 0
78 dsb
79 isb @ Not necessary if followed by eret
80
Christoffer Dall342cd0a2013-01-20 18:28:06 -050081 bx lr
82ENDPROC(__kvm_flush_vm_context)
83
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050084
Christoffer Dall342cd0a2013-01-20 18:28:06 -050085/********************************************************************
86 * Hypervisor world-switch code
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050087 *
88 *
89 * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
Christoffer Dall342cd0a2013-01-20 18:28:06 -050090 */
91ENTRY(__kvm_vcpu_run)
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050092 @ Save the vcpu pointer
93 mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR
94
95 save_host_regs
96
Marc Zyngier1a89dd92013-01-21 19:36:12 -050097 restore_vgic_state
98
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050099 @ Store hardware CP15 state and load guest state
100 read_cp15_state store_to_vcpu = 0
101 write_cp15_state read_from_vcpu = 1
102
103 @ If the host kernel has not been configured with VFPv3 support,
104 @ then it is safer if we deny guests from using it as well.
105#ifdef CONFIG_VFPv3
106 @ Set FPEXC_EN so the guest doesn't trap floating point instructions
107 VFPFMRX r2, FPEXC @ VMRS
108 push {r2}
109 orr r2, r2, #FPEXC_EN
110 VFPFMXR FPEXC, r2 @ VMSR
111#endif
112
113 @ Configure Hyp-role
114 configure_hyp_role vmentry
115
116 @ Trap coprocessor CRx accesses
117 set_hstr vmentry
118 set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
119 set_hdcr vmentry
120
121 @ Write configured ID register into MIDR alias
122 ldr r1, [vcpu, #VCPU_MIDR]
123 mcr p15, 4, r1, c0, c0, 0
124
125 @ Write guest view of MPIDR into VMPIDR
126 ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
127 mcr p15, 4, r1, c0, c0, 5
128
129 @ Set up guest memory translation
130 ldr r1, [vcpu, #VCPU_KVM]
131 add r1, r1, #KVM_VTTBR
132 ldrd r2, r3, [r1]
133 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
134
135 @ We're all done, just restore the GPRs and go to the guest
136 restore_guest_regs
137 clrex @ Clear exclusive monitor
138 eret
139
140__kvm_vcpu_return:
141 /*
142 * return convention:
143 * guest r0, r1, r2 saved on the stack
144 * r0: vcpu pointer
145 * r1: exception code
146 */
147 save_guest_regs
148
149 @ Set VMID == 0
150 mov r2, #0
151 mov r3, #0
152 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
153
154 @ Don't trap coprocessor accesses for host kernel
155 set_hstr vmexit
156 set_hdcr vmexit
157 set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
158
159#ifdef CONFIG_VFPv3
160 @ Save floating point registers we if let guest use them.
161 tst r2, #(HCPTR_TCP(10) | HCPTR_TCP(11))
162 bne after_vfp_restore
163
164 @ Switch VFP/NEON hardware state to the host's
165 add r7, vcpu, #VCPU_VFP_GUEST
166 store_vfp_state r7
167 add r7, vcpu, #VCPU_VFP_HOST
168 ldr r7, [r7]
169 restore_vfp_state r7
170
171after_vfp_restore:
172 @ Restore FPEXC_EN which we clobbered on entry
173 pop {r2}
174 VFPFMXR FPEXC, r2
175#endif
176
177 @ Reset Hyp-role
178 configure_hyp_role vmexit
179
180 @ Let host read hardware MIDR
181 mrc p15, 0, r2, c0, c0, 0
182 mcr p15, 4, r2, c0, c0, 0
183
184 @ Back to hardware MPIDR
185 mrc p15, 0, r2, c0, c0, 5
186 mcr p15, 4, r2, c0, c0, 5
187
188 @ Store guest CP15 state and restore host state
189 read_cp15_state store_to_vcpu = 1
190 write_cp15_state read_from_vcpu = 0
191
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500192 save_vgic_state
193
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500194 restore_host_regs
195 clrex @ Clear exclusive monitor
196 mov r0, r1 @ Return the return code
197 mov r1, #0 @ Clear upper bits in return value
198 bx lr @ return to IOCTL
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500199
200/********************************************************************
201 * Call function in Hyp mode
202 *
203 *
204 * u64 kvm_call_hyp(void *hypfn, ...);
205 *
206 * This is not really a variadic function in the classic C-way and care must
207 * be taken when calling this to ensure parameters are passed in registers
208 * only, since the stack will change between the caller and the callee.
209 *
210 * Call the function with the first argument containing a pointer to the
211 * function you wish to call in Hyp mode, and subsequent arguments will be
212 * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
213 * function pointer can be passed). The function being called must be mapped
214 * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
215 * passed in r0 and r1.
216 *
217 * The calling convention follows the standard AAPCS:
218 * r0 - r3: caller save
219 * r12: caller save
220 * rest: callee save
221 */
222ENTRY(kvm_call_hyp)
223 hvc #0
224 bx lr
225
226/********************************************************************
227 * Hypervisor exception vector and handlers
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500228 *
229 *
230 * The KVM/ARM Hypervisor ABI is defined as follows:
231 *
232 * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
233 * instruction is issued since all traps are disabled when running the host
234 * kernel as per the Hyp-mode initialization at boot time.
235 *
236 * HVC instructions cause a trap to the vector page + offset 0x18 (see hyp_hvc
237 * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
238 * host kernel) and they cause a trap to the vector page + offset 0xc when HVC
239 * instructions are called from within Hyp-mode.
240 *
241 * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
242 * Switching to Hyp mode is done through a simple HVC #0 instruction. The
243 * exception vector code will check that the HVC comes from VMID==0 and if
244 * so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
245 * - r0 contains a pointer to a HYP function
246 * - r1, r2, and r3 contain arguments to the above function.
247 * - The HYP function will be called with its arguments in r0, r1 and r2.
248 * On HYP function return, we return directly to SVC.
249 *
250 * Note that the above is used to execute code in Hyp-mode from a host-kernel
251 * point of view, and is a different concept from performing a world-switch and
252 * executing guest code SVC mode (with a VMID != 0).
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500253 */
254
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500255/* Handle undef, svc, pabt, or dabt by crashing with a user notice */
256.macro bad_exception exception_code, panic_str
257 push {r0-r2}
258 mrrc p15, 6, r0, r1, c2 @ Read VTTBR
259 lsr r1, r1, #16
260 ands r1, r1, #0xff
261 beq 99f
262
263 load_vcpu @ Load VCPU pointer
264 .if \exception_code == ARM_EXCEPTION_DATA_ABORT
265 mrc p15, 4, r2, c5, c2, 0 @ HSR
266 mrc p15, 4, r1, c6, c0, 0 @ HDFAR
267 str r2, [vcpu, #VCPU_HSR]
268 str r1, [vcpu, #VCPU_HxFAR]
269 .endif
270 .if \exception_code == ARM_EXCEPTION_PREF_ABORT
271 mrc p15, 4, r2, c5, c2, 0 @ HSR
272 mrc p15, 4, r1, c6, c0, 2 @ HIFAR
273 str r2, [vcpu, #VCPU_HSR]
274 str r1, [vcpu, #VCPU_HxFAR]
275 .endif
276 mov r1, #\exception_code
277 b __kvm_vcpu_return
278
279 @ We were in the host already. Let's craft a panic-ing return to SVC.
28099: mrs r2, cpsr
281 bic r2, r2, #MODE_MASK
282 orr r2, r2, #SVC_MODE
283THUMB( orr r2, r2, #PSR_T_BIT )
284 msr spsr_cxsf, r2
285 mrs r1, ELR_hyp
286 ldr r2, =BSYM(panic)
287 msr ELR_hyp, r2
288 ldr r0, =\panic_str
289 eret
290.endm
291
292 .text
293
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500294 .align 5
295__kvm_hyp_vector:
296 .globl __kvm_hyp_vector
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500297
298 @ Hyp-mode exception vector
299 W(b) hyp_reset
300 W(b) hyp_undef
301 W(b) hyp_svc
302 W(b) hyp_pabt
303 W(b) hyp_dabt
304 W(b) hyp_hvc
305 W(b) hyp_irq
306 W(b) hyp_fiq
307
308 .align
309hyp_reset:
310 b hyp_reset
311
312 .align
313hyp_undef:
314 bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str
315
316 .align
317hyp_svc:
318 bad_exception ARM_EXCEPTION_HVC, svc_die_str
319
320 .align
321hyp_pabt:
322 bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str
323
324 .align
325hyp_dabt:
326 bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str
327
328 .align
329hyp_hvc:
330 /*
331 * Getting here is either becuase of a trap from a guest or from calling
332 * HVC from the host kernel, which means "switch to Hyp mode".
333 */
334 push {r0, r1, r2}
335
336 @ Check syndrome register
337 mrc p15, 4, r1, c5, c2, 0 @ HSR
338 lsr r0, r1, #HSR_EC_SHIFT
339#ifdef CONFIG_VFPv3
340 cmp r0, #HSR_EC_CP_0_13
341 beq switch_to_guest_vfp
342#endif
343 cmp r0, #HSR_EC_HVC
344 bne guest_trap @ Not HVC instr.
345
346 /*
347 * Let's check if the HVC came from VMID 0 and allow simple
348 * switch to Hyp mode
349 */
350 mrrc p15, 6, r0, r2, c2
351 lsr r2, r2, #16
352 and r2, r2, #0xff
353 cmp r2, #0
354 bne guest_trap @ Guest called HVC
355
356host_switch_to_hyp:
357 pop {r0, r1, r2}
358
359 push {lr}
360 mrs lr, SPSR
361 push {lr}
362
363 mov lr, r0
364 mov r0, r1
365 mov r1, r2
366 mov r2, r3
367
368THUMB( orr lr, #1)
369 blx lr @ Call the HYP function
370
371 pop {lr}
372 msr SPSR_csxf, lr
373 pop {lr}
374 eret
375
376guest_trap:
377 load_vcpu @ Load VCPU pointer to r0
378 str r1, [vcpu, #VCPU_HSR]
379
380 @ Check if we need the fault information
381 lsr r1, r1, #HSR_EC_SHIFT
382 cmp r1, #HSR_EC_IABT
383 mrceq p15, 4, r2, c6, c0, 2 @ HIFAR
384 beq 2f
385 cmp r1, #HSR_EC_DABT
386 bne 1f
387 mrc p15, 4, r2, c6, c0, 0 @ HDFAR
388
3892: str r2, [vcpu, #VCPU_HxFAR]
390
391 /*
392 * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
393 *
394 * Abort on the stage 2 translation for a memory access from a
395 * Non-secure PL1 or PL0 mode:
396 *
397 * For any Access flag fault or Translation fault, and also for any
398 * Permission fault on the stage 2 translation of a memory access
399 * made as part of a translation table walk for a stage 1 translation,
400 * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
401 * is UNKNOWN.
402 */
403
404 /* Check for permission fault, and S1PTW */
405 mrc p15, 4, r1, c5, c2, 0 @ HSR
406 and r0, r1, #HSR_FSC_TYPE
407 cmp r0, #FSC_PERM
408 tsteq r1, #(1 << 7) @ S1PTW
409 mrcne p15, 4, r2, c6, c0, 4 @ HPFAR
410 bne 3f
411
412 /* Resolve IPA using the xFAR */
413 mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR
414 isb
415 mrrc p15, 0, r0, r1, c7 @ PAR
416 tst r0, #1
417 bne 4f @ Failed translation
418 ubfx r2, r0, #12, #20
419 lsl r2, r2, #4
420 orr r2, r2, r1, lsl #24
421
4223: load_vcpu @ Load VCPU pointer to r0
423 str r2, [r0, #VCPU_HPFAR]
424
4251: mov r1, #ARM_EXCEPTION_HVC
426 b __kvm_vcpu_return
427
4284: pop {r0, r1, r2} @ Failed translation, return to guest
429 eret
430
431/*
432 * If VFPv3 support is not available, then we will not switch the VFP
433 * registers; however cp10 and cp11 accesses will still trap and fallback
434 * to the regular coprocessor emulation code, which currently will
435 * inject an undefined exception to the guest.
436 */
437#ifdef CONFIG_VFPv3
438switch_to_guest_vfp:
439 load_vcpu @ Load VCPU pointer to r0
440 push {r3-r7}
441
442 @ NEON/VFP used. Turn on VFP access.
443 set_hcptr vmexit, (HCPTR_TCP(10) | HCPTR_TCP(11))
444
445 @ Switch VFP/NEON hardware state to the guest's
446 add r7, r0, #VCPU_VFP_HOST
447 ldr r7, [r7]
448 store_vfp_state r7
449 add r7, r0, #VCPU_VFP_GUEST
450 restore_vfp_state r7
451
452 pop {r3-r7}
453 pop {r0-r2}
454 eret
455#endif
456
457 .align
458hyp_irq:
459 push {r0, r1, r2}
460 mov r1, #ARM_EXCEPTION_IRQ
461 load_vcpu @ Load VCPU pointer to r0
462 b __kvm_vcpu_return
463
464 .align
465hyp_fiq:
466 b hyp_fiq
467
468 .ltorg
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500469
470__kvm_hyp_code_end:
471 .globl __kvm_hyp_code_end
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500472
473 .section ".rodata"
474
475und_die_str:
476 .ascii "unexpected undefined exception in Hyp mode at: %#08x"
477pabt_die_str:
478 .ascii "unexpected prefetch abort in Hyp mode at: %#08x"
479dabt_die_str:
480 .ascii "unexpected data abort in Hyp mode at: %#08x"
481svc_die_str:
482 .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x"