/* Asm versions of Xen pv-ops, suitable for either direct use or inlining. The inline versions are the same as the direct-use versions, with the pre- and post-amble chopped off. This code is encoded for size rather than absolute efficiency, with a view to being able to inline as much as possible. We only bother with direct forms (ie, vcpu in pda) of the operations here; the indirect forms are better handled in C, since they're generally too large to inline anyway. */ #include #include #include #include #include #include #define RELOC(x, v) .globl x##_reloc; x##_reloc=v #define ENDPATCH(x) .globl x##_end; x##_end=. /* Pseudo-flag used for virtual NMI, which we don't implement yet */ #define XEN_EFLAGS_NMI 0x80000000 #if 1 /* x86-64 does not yet support direct access to percpu variables via a segment override, so we just need to make sure this code never gets used */ #define BUG ud2a #define PER_CPU_VAR(var, off) 0xdeadbeef #endif /* Enable events. This clears the event mask and tests the pending event status with one and operation. If there are pending events, then enter the hypervisor to get them handled. */ ENTRY(xen_irq_enable_direct) BUG /* Unmask events */ movb $0, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask) /* Preempt here doesn't matter because that will deal with any pending interrupts. The pending check may end up being run on the wrong CPU, but that doesn't hurt. */ /* Test for pending */ testb $0xff, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_pending) jz 1f 2: call check_events 1: ENDPATCH(xen_irq_enable_direct) ret ENDPROC(xen_irq_enable_direct) RELOC(xen_irq_enable_direct, 2b+1) /* Disabling events is simply a matter of making the event mask non-zero. */ ENTRY(xen_irq_disable_direct) BUG movb $1, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask) ENDPATCH(xen_irq_disable_direct) ret ENDPROC(xen_irq_disable_direct) RELOC(xen_irq_disable_direct, 0) /* (xen_)save_fl is used to get the current interrupt enable status. Callers expect the status to be in X86_EFLAGS_IF, and other bits may be set in the return value. We take advantage of this by making sure that X86_EFLAGS_IF has the right value (and other bits in that byte are 0), but other bits in the return value are undefined. We need to toggle the state of the bit, because Xen and x86 use opposite senses (mask vs enable). */ ENTRY(xen_save_fl_direct) BUG testb $0xff, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask) setz %ah addb %ah,%ah ENDPATCH(xen_save_fl_direct) ret ENDPROC(xen_save_fl_direct) RELOC(xen_save_fl_direct, 0) /* In principle the caller should be passing us a value return from xen_save_fl_direct, but for robustness sake we test only the X86_EFLAGS_IF flag rather than the whole byte. After setting the interrupt mask state, it checks for unmasked pending events and enters the hypervisor to get them delivered if so. */ ENTRY(xen_restore_fl_direct) BUG testb $X86_EFLAGS_IF>>8, %ah setz PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask) /* Preempt here doesn't matter because that will deal with any pending interrupts. The pending check may end up being run on the wrong CPU, but that doesn't hurt. */ /* check for unmasked and pending */ cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_pending) jz 1f 2: call check_events 1: ENDPATCH(xen_restore_fl_direct) ret ENDPROC(xen_restore_fl_direct) RELOC(xen_restore_fl_direct, 2b+1) /* Force an event check by making a hypercall, but preserve regs before making the call. */ check_events: push %rax push %rcx push %rdx push %rsi push %rdi push %r8 push %r9 push %r10 push %r11 call xen_force_evtchn_callback pop %r11 pop %r10 pop %r9 pop %r8 pop %rdi pop %rsi pop %rdx pop %rcx pop %rax ret ENTRY(xen_adjust_exception_frame) mov 8+0(%rsp),%rcx mov 8+8(%rsp),%r11 ret $16 hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 /* Xen64 iret frame: ss rsp rflags cs rip <-- standard iret frame flags rcx } r11 }<-- pushed by hypercall page rsp -> rax } */ ENTRY(xen_iret) pushq $0 1: jmp hypercall_iret ENDPATCH(xen_iret) RELOC(xen_iret, 1b+1) /* sysexit is not used for 64-bit processes, so it's only ever used to return to 32-bit compat userspace. */ ENTRY(xen_sysexit) pushq $__USER32_DS pushq %rcx pushq $X86_EFLAGS_IF pushq $__USER32_CS pushq %rdx pushq $0 1: jmp hypercall_iret ENDPATCH(xen_sysexit) RELOC(xen_sysexit, 1b+1) ENTRY(xen_sysret64) /* We're already on the usermode stack at this point, but still with the kernel gs, so we can easily switch back */ movq %rsp, %gs:pda_oldrsp movq %gs:pda_kernelstack,%rsp pushq $__USER_DS pushq %gs:pda_oldrsp pushq %r11 pushq $__USER_CS pushq %rcx pushq $VGCF_in_syscall 1: jmp hypercall_iret ENDPATCH(xen_sysret64) RELOC(xen_sysret64, 1b+1) ENTRY(xen_sysret32) /* We're already on the usermode stack at this point, but still with the kernel gs, so we can easily switch back */ movq %rsp, %gs:pda_oldrsp movq %gs:pda_kernelstack, %rsp pushq $__USER32_DS pushq %gs:pda_oldrsp pushq %r11 pushq $__USER32_CS pushq %rcx pushq $VGCF_in_syscall 1: jmp hypercall_iret ENDPATCH(xen_sysret32) RELOC(xen_sysret32, 1b+1) /* Xen handles syscall callbacks much like ordinary exceptions, which means we have: - kernel gs - kernel rsp - an iret-like stack frame on the stack (including rcx and r11): ss rsp rflags cs rip r11 rsp-> rcx In all the entrypoints, we undo all that to make it look like a CPU-generated syscall/sysenter and jump to the normal entrypoint. */ .macro undo_xen_syscall mov 0*8(%rsp),%rcx mov 1*8(%rsp),%r11 mov 5*8(%rsp),%rsp .endm /* Normal 64-bit system call target */ ENTRY(xen_syscall_target) undo_xen_syscall jmp system_call_after_swapgs ENDPROC(xen_syscall_target) #ifdef CONFIG_IA32_EMULATION /* 32-bit compat syscall target */ ENTRY(xen_syscall32_target) undo_xen_syscall jmp ia32_cstar_target ENDPROC(xen_syscall32_target) /* 32-bit compat sysenter target */ ENTRY(xen_sysenter_target) undo_xen_syscall jmp ia32_sysenter_target ENDPROC(xen_sysenter_target) #else /* !CONFIG_IA32_EMULATION */ ENTRY(xen_syscall32_target) ENTRY(xen_sysenter_target) lea 16(%rsp), %rsp /* strip %rcx,%r11 */ mov $-ENOSYS, %rax pushq $VGCF_in_syscall jmp hypercall_iret ENDPROC(xen_syscall32_target) ENDPROC(xen_sysenter_target) #endif /* CONFIG_IA32_EMULATION */