aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPeter Maydell <peter.maydell@linaro.org>2016-05-13 10:42:40 +0100
committerPeter Maydell <peter.maydell@linaro.org>2016-05-13 10:42:40 +0100
commit20c20318f9fb0e64c41202c4cd66a7c599cfeecb (patch)
treecf86847705d548f47f3d548b1473c8eb99b8831e
parentf68419eee9a966f5a915314c43cda6778f976a77 (diff)
parent8b1fe3f439eaa2f0a6ee7737942bb6c405725867 (diff)
Merge remote-tracking branch 'remotes/rth/tags/pull-tcg-20160512' into staging
queued 2.7 patches # gpg: Signature made Fri 13 May 2016 01:08:20 BST using RSA key ID 4DD0279B # gpg: Good signature from "Richard Henderson <rth7680@gmail.com>" # gpg: aka "Richard Henderson <rth@redhat.com>" # gpg: aka "Richard Henderson <rth@twiddle.net>" * remotes/rth/tags/pull-tcg-20160512: (39 commits) cpu-exec: Clean up 'interrupt_request' reloading in cpu_handle_interrupt() cpu-exec: Remove unused 'x86_cpu' and 'env' from cpu_exec() cpu-exec: Move TB execution stuff out of cpu_exec() cpu-exec: Move interrupt handling out of cpu_exec() cpu-exec: Move exception handling out of cpu_exec() cpu-exec: Move halt handling out of cpu_exec() cpu-exec: Remove relic orphaned comment tcg: Remove needless CPUState::current_tb cpu-exec: Move TB chaining into tb_find_fast() tcg: Rework tb_invalidated_flag tcg: Clean up from 'next_tb' cpu-exec: elide more icount code if CONFIG_USER_ONLY tcg: reorganize tb_find_physical loop tcg: code_bitmap and code_write_count are not used by user-mode emulation tcg: Allow goto_tb to any target PC in user mode tcg: Clean up direct block chaining safety checks tcg: Clean up tb_jmp_unlink() tcg: Extract removing of jumps to TB from tb_phys_invalidate() tcg: Rename tb_jmp_remove() to tb_remove_from_jmp_list() tcg: Clarify thread safety check in tb_add_jump() ... Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
-rw-r--r--cpu-exec-common.c2
-rw-r--r--cpu-exec.c519
-rw-r--r--cputlb.c13
-rw-r--r--exec.c2
-rw-r--r--hw/i386/kvmvapic.c3
-rw-r--r--include/exec/exec-all.h108
-rw-r--r--include/qemu/osdep.h14
-rw-r--r--include/qom/cpu.h4
-rw-r--r--qom/cpu.c1
-rw-r--r--target-alpha/cpu.h2
-rw-r--r--target-alpha/translate.c4
-rw-r--r--target-arm/cpu.h2
-rw-r--r--target-arm/translate-a64.c2
-rw-r--r--target-arm/translate.c17
-rw-r--r--target-cris/cpu.h2
-rw-r--r--target-cris/translate.c16
-rw-r--r--target-i386/cpu.h2
-rw-r--r--target-i386/translate.c25
-rw-r--r--target-lm32/cpu.h2
-rw-r--r--target-lm32/translate.c21
-rw-r--r--target-m68k/cpu.h2
-rw-r--r--target-m68k/translate.c18
-rw-r--r--target-microblaze/cpu.h2
-rw-r--r--target-microblaze/translate.c15
-rw-r--r--target-mips/cpu.h2
-rw-r--r--target-mips/translate.c20
-rw-r--r--target-moxie/cpu.h2
-rw-r--r--target-moxie/translate.c21
-rw-r--r--target-openrisc/cpu.h2
-rw-r--r--target-openrisc/translate.c20
-rw-r--r--target-ppc/cpu.h2
-rw-r--r--target-ppc/translate.c20
-rw-r--r--target-s390x/cpu.h2
-rw-r--r--target-s390x/translate.c17
-rw-r--r--target-sh4/cpu.h2
-rw-r--r--target-sh4/translate.c21
-rw-r--r--target-sparc/cpu.h2
-rw-r--r--target-sparc/translate.c24
-rw-r--r--target-tilegx/cpu.h2
-rw-r--r--target-tricore/cpu.h2
-rw-r--r--target-tricore/translate.c20
-rw-r--r--target-unicore32/cpu.h2
-rw-r--r--target-unicore32/translate.c16
-rw-r--r--target-xtensa/cpu.h2
-rw-r--r--target-xtensa/translate.c4
-rw-r--r--tcg/aarch64/tcg-target.inc.c21
-rw-r--r--tcg/arm/tcg-target.inc.c26
-rw-r--r--tcg/i386/tcg-target.inc.c31
-rw-r--r--tcg/ia64/tcg-target.inc.c6
-rw-r--r--tcg/mips/tcg-target.inc.c11
-rw-r--r--tcg/ppc/tcg-target.inc.c28
-rw-r--r--tcg/s390/tcg-target.inc.c19
-rw-r--r--tcg/sparc/tcg-target.inc.c11
-rw-r--r--tcg/tcg-op.h13
-rw-r--r--tcg/tcg.h25
-rw-r--r--tcg/tci/tcg-target.inc.c12
-rw-r--r--tci.c11
-rw-r--r--trace-events2
-rw-r--r--translate-all.c354
59 files changed, 917 insertions, 656 deletions
diff --git a/cpu-exec-common.c b/cpu-exec-common.c
index 1b1731cd83..6bdda6b6b0 100644
--- a/cpu-exec-common.c
+++ b/cpu-exec-common.c
@@ -68,7 +68,6 @@ void cpu_reloading_memory_map(void)
void cpu_loop_exit(CPUState *cpu)
{
- cpu->current_tb = NULL;
siglongjmp(cpu->jmp_env, 1);
}
@@ -77,6 +76,5 @@ void cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc)
if (pc) {
cpu_restore_state(cpu, pc);
}
- cpu->current_tb = NULL;
siglongjmp(cpu->jmp_env, 1);
}
diff --git a/cpu-exec.c b/cpu-exec.c
index bbfcbfb543..14df1aacf4 100644
--- a/cpu-exec.c
+++ b/cpu-exec.c
@@ -136,7 +136,9 @@ static void init_delay_params(SyncClocks *sc, const CPUState *cpu)
static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb)
{
CPUArchState *env = cpu->env_ptr;
- uintptr_t next_tb;
+ uintptr_t ret;
+ TranslationBlock *last_tb;
+ int tb_exit;
uint8_t *tb_ptr = itb->tc_ptr;
qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc,
@@ -160,118 +162,125 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb)
#endif /* DEBUG_DISAS */
cpu->can_do_io = !use_icount;
- next_tb = tcg_qemu_tb_exec(env, tb_ptr);
+ ret = tcg_qemu_tb_exec(env, tb_ptr);
cpu->can_do_io = 1;
- trace_exec_tb_exit((void *) (next_tb & ~TB_EXIT_MASK),
- next_tb & TB_EXIT_MASK);
+ last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK);
+ tb_exit = ret & TB_EXIT_MASK;
+ trace_exec_tb_exit(last_tb, tb_exit);
- if ((next_tb & TB_EXIT_MASK) > TB_EXIT_IDX1) {
+ if (tb_exit > TB_EXIT_IDX1) {
/* We didn't start executing this TB (eg because the instruction
* counter hit zero); we must restore the guest PC to the address
* of the start of the TB.
*/
CPUClass *cc = CPU_GET_CLASS(cpu);
- TranslationBlock *tb = (TranslationBlock *)(next_tb & ~TB_EXIT_MASK);
- qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc,
+ qemu_log_mask_and_addr(CPU_LOG_EXEC, last_tb->pc,
"Stopped execution of TB chain before %p ["
TARGET_FMT_lx "] %s\n",
- itb->tc_ptr, itb->pc, lookup_symbol(itb->pc));
+ last_tb->tc_ptr, last_tb->pc,
+ lookup_symbol(last_tb->pc));
if (cc->synchronize_from_tb) {
- cc->synchronize_from_tb(cpu, tb);
+ cc->synchronize_from_tb(cpu, last_tb);
} else {
assert(cc->set_pc);
- cc->set_pc(cpu, tb->pc);
+ cc->set_pc(cpu, last_tb->pc);
}
}
- if ((next_tb & TB_EXIT_MASK) == TB_EXIT_REQUESTED) {
+ if (tb_exit == TB_EXIT_REQUESTED) {
/* We were asked to stop executing TBs (probably a pending
* interrupt. We've now stopped, so clear the flag.
*/
cpu->tcg_exit_req = 0;
}
- return next_tb;
+ return ret;
}
+#ifndef CONFIG_USER_ONLY
/* Execute the code without caching the generated code. An interpreter
could be used if available. */
static void cpu_exec_nocache(CPUState *cpu, int max_cycles,
TranslationBlock *orig_tb, bool ignore_icount)
{
TranslationBlock *tb;
+ bool old_tb_flushed;
/* Should never happen.
We only end up here when an existing TB is too long. */
if (max_cycles > CF_COUNT_MASK)
max_cycles = CF_COUNT_MASK;
+ old_tb_flushed = cpu->tb_flushed;
+ cpu->tb_flushed = false;
tb = tb_gen_code(cpu, orig_tb->pc, orig_tb->cs_base, orig_tb->flags,
max_cycles | CF_NOCACHE
| (ignore_icount ? CF_IGNORE_ICOUNT : 0));
- tb->orig_tb = tcg_ctx.tb_ctx.tb_invalidated_flag ? NULL : orig_tb;
- cpu->current_tb = tb;
+ tb->orig_tb = cpu->tb_flushed ? NULL : orig_tb;
+ cpu->tb_flushed |= old_tb_flushed;
/* execute the generated code */
trace_exec_tb_nocache(tb, tb->pc);
cpu_tb_exec(cpu, tb);
- cpu->current_tb = NULL;
tb_phys_invalidate(tb, -1);
tb_free(tb);
}
+#endif
static TranslationBlock *tb_find_physical(CPUState *cpu,
target_ulong pc,
target_ulong cs_base,
- uint64_t flags)
+ uint32_t flags)
{
CPUArchState *env = (CPUArchState *)cpu->env_ptr;
- TranslationBlock *tb, **ptb1;
+ TranslationBlock *tb, **tb_hash_head, **ptb1;
unsigned int h;
tb_page_addr_t phys_pc, phys_page1;
- target_ulong virt_page2;
-
- tcg_ctx.tb_ctx.tb_invalidated_flag = 0;
/* find translated block using physical mappings */
phys_pc = get_page_addr_code(env, pc);
phys_page1 = phys_pc & TARGET_PAGE_MASK;
h = tb_phys_hash_func(phys_pc);
- ptb1 = &tcg_ctx.tb_ctx.tb_phys_hash[h];
- for(;;) {
- tb = *ptb1;
- if (!tb) {
- return NULL;
- }
+
+ /* Start at head of the hash entry */
+ ptb1 = tb_hash_head = &tcg_ctx.tb_ctx.tb_phys_hash[h];
+ tb = *ptb1;
+
+ while (tb) {
if (tb->pc == pc &&
tb->page_addr[0] == phys_page1 &&
tb->cs_base == cs_base &&
tb->flags == flags) {
- /* check next page if needed */
- if (tb->page_addr[1] != -1) {
- tb_page_addr_t phys_page2;
- virt_page2 = (pc & TARGET_PAGE_MASK) +
- TARGET_PAGE_SIZE;
- phys_page2 = get_page_addr_code(env, virt_page2);
+ if (tb->page_addr[1] == -1) {
+ /* done, we have a match */
+ break;
+ } else {
+ /* check next page if needed */
+ target_ulong virt_page2 = (pc & TARGET_PAGE_MASK) +
+ TARGET_PAGE_SIZE;
+ tb_page_addr_t phys_page2 = get_page_addr_code(env, virt_page2);
+
if (tb->page_addr[1] == phys_page2) {
break;
}
- } else {
- break;
}
}
+
ptb1 = &tb->phys_hash_next;
+ tb = *ptb1;
}
- /* Move the TB to the head of the list */
- *ptb1 = tb->phys_hash_next;
- tb->phys_hash_next = tcg_ctx.tb_ctx.tb_phys_hash[h];
- tcg_ctx.tb_ctx.tb_phys_hash[h] = tb;
+ if (tb) {
+ /* Move the TB to the head of the list */
+ *ptb1 = tb->phys_hash_next;
+ tb->phys_hash_next = *tb_hash_head;
+ *tb_hash_head = tb;
+ }
return tb;
}
static TranslationBlock *tb_find_slow(CPUState *cpu,
target_ulong pc,
target_ulong cs_base,
- uint64_t flags)
+ uint32_t flags)
{
TranslationBlock *tb;
@@ -309,26 +318,63 @@ found:
return tb;
}
-static inline TranslationBlock *tb_find_fast(CPUState *cpu)
+static inline TranslationBlock *tb_find_fast(CPUState *cpu,
+ TranslationBlock **last_tb,
+ int tb_exit)
{
CPUArchState *env = (CPUArchState *)cpu->env_ptr;
TranslationBlock *tb;
target_ulong cs_base, pc;
- int flags;
+ uint32_t flags;
/* we record a subset of the CPU state. It will
always be the same before a given translated block
is executed. */
cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
+ tb_lock();
tb = cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)];
if (unlikely(!tb || tb->pc != pc || tb->cs_base != cs_base ||
tb->flags != flags)) {
tb = tb_find_slow(cpu, pc, cs_base, flags);
}
+ if (cpu->tb_flushed) {
+ /* Ensure that no TB jump will be modified as the
+ * translation buffer has been flushed.
+ */
+ *last_tb = NULL;
+ cpu->tb_flushed = false;
+ }
+ /* See if we can patch the calling TB. */
+ if (*last_tb && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
+ tb_add_jump(*last_tb, tb_exit, tb);
+ }
+ tb_unlock();
return tb;
}
-static void cpu_handle_debug_exception(CPUState *cpu)
+static inline bool cpu_handle_halt(CPUState *cpu)
+{
+ if (cpu->halted) {
+#if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY)
+ if ((cpu->interrupt_request & CPU_INTERRUPT_POLL)
+ && replay_interrupt()) {
+ X86CPU *x86_cpu = X86_CPU(cpu);
+ apic_poll_irq(x86_cpu->apic_state);
+ cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL);
+ }
+#endif
+ if (!cpu_has_work(cpu)) {
+ current_cpu = NULL;
+ return true;
+ }
+
+ cpu->halted = 0;
+ }
+
+ return false;
+}
+
+static inline void cpu_handle_debug_exception(CPUState *cpu)
{
CPUClass *cc = CPU_GET_CLASS(cpu);
CPUWatchpoint *wp;
@@ -342,37 +388,197 @@ static void cpu_handle_debug_exception(CPUState *cpu)
cc->debug_excp_handler(cpu);
}
+static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
+{
+ if (cpu->exception_index >= 0) {
+ if (cpu->exception_index >= EXCP_INTERRUPT) {
+ /* exit request from the cpu execution loop */
+ *ret = cpu->exception_index;
+ if (*ret == EXCP_DEBUG) {
+ cpu_handle_debug_exception(cpu);
+ }
+ cpu->exception_index = -1;
+ return true;
+ } else {
+#if defined(CONFIG_USER_ONLY)
+ /* if user mode only, we simulate a fake exception
+ which will be handled outside the cpu execution
+ loop */
+#if defined(TARGET_I386)
+ CPUClass *cc = CPU_GET_CLASS(cpu);
+ cc->do_interrupt(cpu);
+#endif
+ *ret = cpu->exception_index;
+ cpu->exception_index = -1;
+ return true;
+#else
+ if (replay_exception()) {
+ CPUClass *cc = CPU_GET_CLASS(cpu);
+ cc->do_interrupt(cpu);
+ cpu->exception_index = -1;
+ } else if (!replay_has_interrupt()) {
+ /* give a chance to iothread in replay mode */
+ *ret = EXCP_INTERRUPT;
+ return true;
+ }
+#endif
+ }
+#ifndef CONFIG_USER_ONLY
+ } else if (replay_has_exception()
+ && cpu->icount_decr.u16.low + cpu->icount_extra == 0) {
+ /* try to cause an exception pending in the log */
+ TranslationBlock *last_tb = NULL; /* Avoid chaining TBs */
+ cpu_exec_nocache(cpu, 1, tb_find_fast(cpu, &last_tb, 0), true);
+ *ret = -1;
+ return true;
+#endif
+ }
+
+ return false;
+}
+
+static inline void cpu_handle_interrupt(CPUState *cpu,
+ TranslationBlock **last_tb)
+{
+ CPUClass *cc = CPU_GET_CLASS(cpu);
+ int interrupt_request = cpu->interrupt_request;
+
+ if (unlikely(interrupt_request)) {
+ if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) {
+ /* Mask out external interrupts for this step. */
+ interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK;
+ }
+ if (interrupt_request & CPU_INTERRUPT_DEBUG) {
+ cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG;
+ cpu->exception_index = EXCP_DEBUG;
+ cpu_loop_exit(cpu);
+ }
+ if (replay_mode == REPLAY_MODE_PLAY && !replay_has_interrupt()) {
+ /* Do nothing */
+ } else if (interrupt_request & CPU_INTERRUPT_HALT) {
+ replay_interrupt();
+ cpu->interrupt_request &= ~CPU_INTERRUPT_HALT;
+ cpu->halted = 1;
+ cpu->exception_index = EXCP_HLT;
+ cpu_loop_exit(cpu);
+ }
+#if defined(TARGET_I386)
+ else if (interrupt_request & CPU_INTERRUPT_INIT) {
+ X86CPU *x86_cpu = X86_CPU(cpu);
+ CPUArchState *env = &x86_cpu->env;
+ replay_interrupt();
+ cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0);
+ do_cpu_init(x86_cpu);
+ cpu->exception_index = EXCP_HALTED;
+ cpu_loop_exit(cpu);
+ }
+#else
+ else if (interrupt_request & CPU_INTERRUPT_RESET) {
+ replay_interrupt();
+ cpu_reset(cpu);
+ cpu_loop_exit(cpu);
+ }
+#endif
+ /* The target hook has 3 exit conditions:
+ False when the interrupt isn't processed,
+ True when it is, and we should restart on a new TB,
+ and via longjmp via cpu_loop_exit. */
+ else {
+ replay_interrupt();
+ if (cc->cpu_exec_interrupt(cpu, interrupt_request)) {
+ *last_tb = NULL;
+ }
+ /* The target hook may have updated the 'cpu->interrupt_request';
+ * reload the 'interrupt_request' value */
+ interrupt_request = cpu->interrupt_request;
+ }
+ if (interrupt_request & CPU_INTERRUPT_EXITTB) {
+ cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB;
+ /* ensure that no TB jump will be modified as
+ the program flow was changed */
+ *last_tb = NULL;
+ }
+ }
+ if (unlikely(cpu->exit_request || replay_has_interrupt())) {
+ cpu->exit_request = 0;
+ cpu->exception_index = EXCP_INTERRUPT;
+ cpu_loop_exit(cpu);
+ }
+}
+
+static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb,
+ TranslationBlock **last_tb, int *tb_exit,
+ SyncClocks *sc)
+{
+ uintptr_t ret;
+
+ if (unlikely(cpu->exit_request)) {
+ return;
+ }
+
+ trace_exec_tb(tb, tb->pc);
+ ret = cpu_tb_exec(cpu, tb);
+ *last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK);
+ *tb_exit = ret & TB_EXIT_MASK;
+ switch (*tb_exit) {
+ case TB_EXIT_REQUESTED:
+ /* Something asked us to stop executing
+ * chained TBs; just continue round the main
+ * loop. Whatever requested the exit will also
+ * have set something else (eg exit_request or
+ * interrupt_request) which we will handle
+ * next time around the loop. But we need to
+ * ensure the tcg_exit_req read in generated code
+ * comes before the next read of cpu->exit_request
+ * or cpu->interrupt_request.
+ */
+ smp_rmb();
+ *last_tb = NULL;
+ break;
+ case TB_EXIT_ICOUNT_EXPIRED:
+ {
+ /* Instruction counter expired. */
+#ifdef CONFIG_USER_ONLY
+ abort();
+#else
+ int insns_left = cpu->icount_decr.u32;
+ if (cpu->icount_extra && insns_left >= 0) {
+ /* Refill decrementer and continue execution. */
+ cpu->icount_extra += insns_left;
+ insns_left = MIN(0xffff, cpu->icount_extra);
+ cpu->icount_extra -= insns_left;
+ cpu->icount_decr.u16.low = insns_left;
+ } else {
+ if (insns_left > 0) {
+ /* Execute remaining instructions. */
+ cpu_exec_nocache(cpu, insns_left, *last_tb, false);
+ align_clocks(sc, cpu);
+ }
+ cpu->exception_index = EXCP_INTERRUPT;
+ *last_tb = NULL;
+ cpu_loop_exit(cpu);
+ }
+ break;
+#endif
+ }
+ default:
+ break;
+ }
+}
+
/* main execution loop */
int cpu_exec(CPUState *cpu)
{
CPUClass *cc = CPU_GET_CLASS(cpu);
-#ifdef TARGET_I386
- X86CPU *x86_cpu = X86_CPU(cpu);
- CPUArchState *env = &x86_cpu->env;
-#endif
- int ret, interrupt_request;
- TranslationBlock *tb;
- uintptr_t next_tb;
+ int ret;
SyncClocks sc;
/* replay_interrupt may need current_cpu */
current_cpu = cpu;
- if (cpu->halted) {
-#if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY)
- if ((cpu->interrupt_request & CPU_INTERRUPT_POLL)
- && replay_interrupt()) {
- apic_poll_irq(x86_cpu->apic_state);
- cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL);
- }
-#endif
- if (!cpu_has_work(cpu)) {
- current_cpu = NULL;
- return EXCP_HALTED;
- }
-
- cpu->halted = 0;
+ if (cpu_handle_halt(cpu)) {
+ return EXCP_HALTED;
}
atomic_mb_set(&tcg_current_cpu, cpu);
@@ -391,185 +597,26 @@ int cpu_exec(CPUState *cpu)
*/
init_delay_params(&sc, cpu);
- /* prepare setjmp context for exception handling */
for(;;) {
+ TranslationBlock *tb, *last_tb;
+ int tb_exit = 0;
+
+ /* prepare setjmp context for exception handling */
if (sigsetjmp(cpu->jmp_env, 0) == 0) {
/* if an exception is pending, we execute it here */
- if (cpu->exception_index >= 0) {
- if (cpu->exception_index >= EXCP_INTERRUPT) {
- /* exit request from the cpu execution loop */
- ret = cpu->exception_index;
- if (ret == EXCP_DEBUG) {
- cpu_handle_debug_exception(cpu);
- }
- cpu->exception_index = -1;
- break;
- } else {
-#if defined(CONFIG_USER_ONLY)
- /* if user mode only, we simulate a fake exception
- which will be handled outside the cpu execution
- loop */
-#if defined(TARGET_I386)
- cc->do_interrupt(cpu);
-#endif
- ret = cpu->exception_index;
- cpu->exception_index = -1;
- break;
-#else
- if (replay_exception()) {
- cc->do_interrupt(cpu);
- cpu->exception_index = -1;
- } else if (!replay_has_interrupt()) {
- /* give a chance to iothread in replay mode */
- ret = EXCP_INTERRUPT;
- break;
- }
-#endif
- }
- } else if (replay_has_exception()
- && cpu->icount_decr.u16.low + cpu->icount_extra == 0) {
- /* try to cause an exception pending in the log */
- cpu_exec_nocache(cpu, 1, tb_find_fast(cpu), true);
- ret = -1;
+ if (cpu_handle_exception(cpu, &ret)) {
break;
}
- next_tb = 0; /* force lookup of first TB */
+ last_tb = NULL; /* forget the last executed TB after exception */
+ cpu->tb_flushed = false; /* reset before first TB lookup */
for(;;) {
- interrupt_request = cpu->interrupt_request;
- if (unlikely(interrupt_request)) {
- if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) {
- /* Mask out external interrupts for this step. */
- interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK;
- }
- if (interrupt_request & CPU_INTERRUPT_DEBUG) {
- cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG;
- cpu->exception_index = EXCP_DEBUG;
- cpu_loop_exit(cpu);
- }
- if (replay_mode == REPLAY_MODE_PLAY
- && !replay_has_interrupt()) {
- /* Do nothing */
- } else if (interrupt_request & CPU_INTERRUPT_HALT) {
- replay_interrupt();
- cpu->interrupt_request &= ~CPU_INTERRUPT_HALT;
- cpu->halted = 1;
- cpu->exception_index = EXCP_HLT;
- cpu_loop_exit(cpu);
- }
-#if defined(TARGET_I386)
- else if (interrupt_request & CPU_INTERRUPT_INIT) {
- replay_interrupt();
- cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0);
- do_cpu_init(x86_cpu);
- cpu->exception_index = EXCP_HALTED;
- cpu_loop_exit(cpu);
- }
-#else
- else if (interrupt_request & CPU_INTERRUPT_RESET) {
- replay_interrupt();
- cpu_reset(cpu);
- cpu_loop_exit(cpu);
- }
-#endif
- /* The target hook has 3 exit conditions:
- False when the interrupt isn't processed,
- True when it is, and we should restart on a new TB,
- and via longjmp via cpu_loop_exit. */
- else {
- replay_interrupt();
- if (cc->cpu_exec_interrupt(cpu, interrupt_request)) {
- next_tb = 0;
- }
- }
- /* Don't use the cached interrupt_request value,
- do_interrupt may have updated the EXITTB flag. */
- if (cpu->interrupt_request & CPU_INTERRUPT_EXITTB) {
- cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB;
- /* ensure that no TB jump will be modified as
- the program flow was changed */
- next_tb = 0;
- }
- }
- if (unlikely(cpu->exit_request
- || replay_has_interrupt())) {
- cpu->exit_request = 0;
- cpu->exception_index = EXCP_INTERRUPT;
- cpu_loop_exit(cpu);
- }
- tb_lock();
- tb = tb_find_fast(cpu);
- /* Note: we do it here to avoid a gcc bug on Mac OS X when
- doing it in tb_find_slow */
- if (tcg_ctx.tb_ctx.tb_invalidated_flag) {
- /* as some TB could have been invalidated because
- of memory exceptions while generating the code, we
- must recompute the hash index here */
- next_tb = 0;
- tcg_ctx.tb_ctx.tb_invalidated_flag = 0;
- }
- /* see if we can patch the calling TB. When the TB
- spans two pages, we cannot safely do a direct
- jump. */
- if (next_tb != 0 && tb->page_addr[1] == -1
- && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
- tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK),
- next_tb & TB_EXIT_MASK, tb);
- }
- tb_unlock();
- if (likely(!cpu->exit_request)) {
- trace_exec_tb(tb, tb->pc);
- /* execute the generated code */
- cpu->current_tb = tb;
- next_tb = cpu_tb_exec(cpu, tb);
- cpu->current_tb = NULL;
- switch (next_tb & TB_EXIT_MASK) {
- case TB_EXIT_REQUESTED:
- /* Something asked us to stop executing
- * chained TBs; just continue round the main
- * loop. Whatever requested the exit will also
- * have set something else (eg exit_request or
- * interrupt_request) which we will handle
- * next time around the loop. But we need to
- * ensure the tcg_exit_req read in generated code
- * comes before the next read of cpu->exit_request
- * or cpu->interrupt_request.
- */
- smp_rmb();
- next_tb = 0;
- break;
- case TB_EXIT_ICOUNT_EXPIRED:
- {
- /* Instruction counter expired. */
- int insns_left = cpu->icount_decr.u32;
- if (cpu->icount_extra && insns_left >= 0) {
- /* Refill decrementer and continue execution. */
- cpu->icount_extra += insns_left;
- insns_left = MIN(0xffff, cpu->icount_extra);
- cpu->icount_extra -= insns_left;
- cpu->icount_decr.u16.low = insns_left;
- } else {
- if (insns_left > 0) {
- /* Execute remaining instructions. */
- tb = (TranslationBlock *)(next_tb & ~TB_EXIT_MASK);
- cpu_exec_nocache(cpu, insns_left, tb, false);
- align_clocks(&sc, cpu);
- }
- cpu->exception_index = EXCP_INTERRUPT;
- next_tb = 0;
- cpu_loop_exit(cpu);
- }
- break;
- }
- default:
- break;
- }
- }
+ cpu_handle_interrupt(cpu, &last_tb);
+ tb = tb_find_fast(cpu, &last_tb, tb_exit);
+ cpu_loop_exec_tb(cpu, tb, &last_tb, &tb_exit, &sc);
/* Try to align the host and virtual clocks
if the guest is in advance */
align_clocks(&sc, cpu);
- /* reset soft MMU for next block (it can currently
- only be set by a memory fault) */
} /* for(;;) */
} else {
#if defined(__clang__) || !QEMU_GNUC_PREREQ(4, 6)
@@ -579,18 +626,10 @@ int cpu_exec(CPUState *cpu)
* Newer versions of gcc would complain about this code (-Wclobbered). */
cpu = current_cpu;
cc = CPU_GET_CLASS(cpu);
-#ifdef TARGET_I386
- x86_cpu = X86_CPU(cpu);
- env = &x86_cpu->env;
-#endif
#else /* buggy compiler */
/* Assert that the compiler does not smash local variables. */
g_assert(cpu == current_cpu);
g_assert(cc == CPU_GET_CLASS(cpu));
-#ifdef TARGET_I386
- g_assert(x86_cpu == X86_CPU(cpu));
- g_assert(env == &x86_cpu->env);
-#endif
#endif /* buggy compiler */
cpu->can_do_io = 1;
tb_lock_reset();
diff --git a/cputlb.c b/cputlb.c
index 466663b56c..167280ae96 100644
--- a/cputlb.c
+++ b/cputlb.c
@@ -76,10 +76,6 @@ void tlb_flush(CPUState *cpu, int flush_global)
tlb_debug("(%d)\n", flush_global);
- /* must reset current TB so that interrupts cannot modify the
- links while we are modifying them */
- cpu->current_tb = NULL;
-
memset(env->tlb_table, -1, sizeof(env->tlb_table));
memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
@@ -95,9 +91,6 @@ static inline void v_tlb_flush_by_mmuidx(CPUState *cpu, va_list argp)
CPUArchState *env = cpu->env_ptr;
tlb_debug("start\n");
- /* must reset current TB so that interrupts cannot modify the
- links while we are modifying them */
- cpu->current_tb = NULL;
for (;;) {
int mmu_idx = va_arg(argp, int);
@@ -152,9 +145,6 @@ void tlb_flush_page(CPUState *cpu, target_ulong addr)
tlb_flush(cpu, 1);
return;
}
- /* must reset current TB so that interrupts cannot modify the
- links while we are modifying them */
- cpu->current_tb = NULL;
addr &= TARGET_PAGE_MASK;
i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
@@ -193,9 +183,6 @@ void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...)
va_end(argp);
return;
}
- /* must reset current TB so that interrupts cannot modify the
- links while we are modifying them */
- cpu->current_tb = NULL;
addr &= TARGET_PAGE_MASK;
i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
diff --git a/exec.c b/exec.c
index c4f9036184..ee45472cab 100644
--- a/exec.c
+++ b/exec.c
@@ -2087,7 +2087,7 @@ static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
target_ulong pc, cs_base;
target_ulong vaddr;
CPUWatchpoint *wp;
- int cpu_flags;
+ uint32_t cpu_flags;
if (cpu->watchpoint_hit) {
/* We re-entered the check after replacing the TB. Now raise
diff --git a/hw/i386/kvmvapic.c b/hw/i386/kvmvapic.c
index c69f374049..f14445d4fb 100644
--- a/hw/i386/kvmvapic.c
+++ b/hw/i386/kvmvapic.c
@@ -397,7 +397,7 @@ static void patch_instruction(VAPICROMState *s, X86CPU *cpu, target_ulong ip)
uint32_t imm32;
target_ulong current_pc = 0;
target_ulong current_cs_base = 0;
- int current_flags = 0;
+ uint32_t current_flags = 0;
if (smp_cpus == 1) {
handlers = &s->rom_state.up;
@@ -446,7 +446,6 @@ static void patch_instruction(VAPICROMState *s, X86CPU *cpu, target_ulong ip)
resume_all_vcpus();
if (!kvm_enabled()) {
- cs->current_tb = NULL;
tb_gen_code(cs, current_pc, current_cs_base, current_flags, 1);
cpu_resume_from_signal(cs, NULL);
}
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 736209505a..85528f9941 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -76,7 +76,8 @@ bool cpu_restore_state(CPUState *cpu, uintptr_t searched_pc);
void QEMU_NORETURN cpu_resume_from_signal(CPUState *cpu, void *puc);
void QEMU_NORETURN cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
TranslationBlock *tb_gen_code(CPUState *cpu,
- target_ulong pc, target_ulong cs_base, int flags,
+ target_ulong pc, target_ulong cs_base,
+ uint32_t flags,
int cflags);
void cpu_exec_init(CPUState *cpu, Error **errp);
void QEMU_NORETURN cpu_loop_exit(CPUState *cpu);
@@ -229,13 +230,14 @@ static inline void tlb_flush_by_mmuidx(CPUState *cpu, ...)
|| defined(__sparc__) || defined(__aarch64__) \
|| defined(__s390x__) || defined(__mips__) \
|| defined(CONFIG_TCG_INTERPRETER)
+/* NOTE: Direct jump patching must be atomic to be thread-safe. */
#define USE_DIRECT_JUMP
#endif
struct TranslationBlock {
target_ulong pc; /* simulated PC corresponding to this block (EIP + CS base) */
target_ulong cs_base; /* CS base for this block */
- uint64_t flags; /* flags defining in which context the code was generated */
+ uint32_t flags; /* flags defining in which context the code was generated */
uint16_t size; /* size of target code for this block (1 <=
size <= TARGET_PAGE_SIZE) */
uint16_t icount;
@@ -257,20 +259,34 @@ struct TranslationBlock {
struct TranslationBlock *page_next[2];
tb_page_addr_t page_addr[2];
- /* the following data are used to directly call another TB from
- the code of this one. */
- uint16_t tb_next_offset[2]; /* offset of original jump target */
+ /* The following data are used to directly call another TB from
+ * the code of this one. This can be done either by emitting direct or
+ * indirect native jump instructions. These jumps are reset so that the TB
+ * just continue its execution. The TB can be linked to another one by
+ * setting one of the jump targets (or patching the jump instruction). Only
+ * two of such jumps are supported.
+ */
+ uint16_t jmp_reset_offset[2]; /* offset of original jump target */
+#define TB_JMP_RESET_OFFSET_INVALID 0xffff /* indicates no jump generated */
#ifdef USE_DIRECT_JUMP
- uint16_t tb_jmp_offset[2]; /* offset of jump instruction */
+ uint16_t jmp_insn_offset[2]; /* offset of native jump instruction */
#else
- uintptr_t tb_next[2]; /* address of jump generated code */
+ uintptr_t jmp_target_addr[2]; /* target address for indirect jump */
#endif
- /* list of TBs jumping to this one. This is a circular list using
- the two least significant bits of the pointers to tell what is
- the next pointer: 0 = jmp_next[0], 1 = jmp_next[1], 2 =
- jmp_first */
- struct TranslationBlock *jmp_next[2];
- struct TranslationBlock *jmp_first;
+ /* Each TB has an assosiated circular list of TBs jumping to this one.
+ * jmp_list_first points to the first TB jumping to this one.
+ * jmp_list_next is used to point to the next TB in a list.
+ * Since each TB can have two jumps, it can participate in two lists.
+ * jmp_list_first and jmp_list_next are 4-byte aligned pointers to a
+ * TranslationBlock structure, but the two least significant bits of
+ * them are used to encode which data field of the pointed TB should
+ * be used to traverse the list further from that TB:
+ * 0 => jmp_list_next[0], 1 => jmp_list_next[1], 2 => jmp_list_first.
+ * In other words, 0/1 tells which jump is used in the pointed TB,
+ * and 2 means that this is a pointer back to the target TB of this list.
+ */
+ uintptr_t jmp_list_next[2];
+ uintptr_t jmp_list_first;
};
#include "qemu/thread.h"
@@ -288,8 +304,6 @@ struct TBContext {
/* statistics */
int tb_flush_count;
int tb_phys_invalidate_count;
-
- int tb_invalidated_flag;
};
void tb_free(TranslationBlock *tb);
@@ -302,7 +316,7 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
{
/* patch the branch destination */
- *(uint32_t *)jmp_addr = addr - (jmp_addr + 4);
+ atomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
/* no need to flush icache explicitly */
}
#elif defined(_ARCH_PPC)
@@ -312,7 +326,7 @@ void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr);
static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
{
/* patch the branch destination */
- stl_le_p((void*)jmp_addr, addr - (jmp_addr + 4));
+ atomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
/* no need to flush icache explicitly */
}
#elif defined(__s390x__)
@@ -320,36 +334,15 @@ static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
{
/* patch the branch destination */
intptr_t disp = addr - (jmp_addr - 2);
- stl_be_p((void*)jmp_addr, disp / 2);
+ atomic_set((int32_t *)jmp_addr, disp / 2);
/* no need to flush icache explicitly */
}
#elif defined(__aarch64__)
void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr);
#define tb_set_jmp_target1 aarch64_tb_set_jmp_target
#elif defined(__arm__)
-static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
-{
-#if !QEMU_GNUC_PREREQ(4, 1)
- register unsigned long _beg __asm ("a1");
- register unsigned long _end __asm ("a2");
- register unsigned long _flg __asm ("a3");
-#endif
-
- /* we could use a ldr pc, [pc, #-4] kind of branch and avoid the flush */
- *(uint32_t *)jmp_addr =
- (*(uint32_t *)jmp_addr & ~0xffffff)
- | (((addr - (jmp_addr + 8)) >> 2) & 0xffffff);
-
-#if QEMU_GNUC_PREREQ(4, 1)
- __builtin___clear_cache((char *) jmp_addr, (char *) jmp_addr + 4);
-#else
- /* flush icache */
- _beg = jmp_addr;
- _end = jmp_addr + 4;
- _flg = 0;
- __asm __volatile__ ("swi 0x9f0002" : : "r" (_beg), "r" (_end), "r" (_flg));
-#endif
-}
+void arm_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr);
+#define tb_set_jmp_target1 arm_tb_set_jmp_target
#elif defined(__sparc__) || defined(__mips__)
void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr);
#else
@@ -359,7 +352,7 @@ void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr);
static inline void tb_set_jmp_target(TranslationBlock *tb,
int n, uintptr_t addr)
{
- uint16_t offset = tb->tb_jmp_offset[n];
+ uint16_t offset = tb->jmp_insn_offset[n];
tb_set_jmp_target1((uintptr_t)(tb->tc_ptr + offset), addr);
}
@@ -369,7 +362,7 @@ static inline void tb_set_jmp_target(TranslationBlock *tb,
static inline void tb_set_jmp_target(TranslationBlock *tb,
int n, uintptr_t addr)
{
- tb->tb_next[n] = addr;
+ tb->jmp_target_addr[n] = addr;
}
#endif
@@ -377,20 +370,23 @@ static inline void tb_set_jmp_target(TranslationBlock *tb,
static inline void tb_add_jump(TranslationBlock *tb, int n,
TranslationBlock *tb_next)
{
- /* NOTE: this test is only needed for thread safety */
- if (!tb->jmp_next[n]) {
- qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc,
- "Linking TBs %p [" TARGET_FMT_lx
- "] index %d -> %p [" TARGET_FMT_lx "]\n",
- tb->tc_ptr, tb->pc, n,
- tb_next->tc_ptr, tb_next->pc);
- /* patch the native jump address */
- tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc_ptr);
-
- /* add in TB jmp circular list */
- tb->jmp_next[n] = tb_next->jmp_first;
- tb_next->jmp_first = (TranslationBlock *)((uintptr_t)(tb) | (n));
+ if (tb->jmp_list_next[n]) {
+ /* Another thread has already done this while we were
+ * outside of the lock; nothing to do in this case */
+ return;
}
+ qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc,
+ "Linking TBs %p [" TARGET_FMT_lx
+ "] index %d -> %p [" TARGET_FMT_lx "]\n",
+ tb->tc_ptr, tb->pc, n,
+ tb_next->tc_ptr, tb_next->pc);
+
+ /* patch the native jump address */
+ tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc_ptr);
+
+ /* add in TB jmp circular list */
+ tb->jmp_list_next[n] = tb_next->jmp_list_first;
+ tb_next->jmp_list_first = (uintptr_t)tb | n;
}
/* GETRA is the true target of the return instruction that we'll execute,
diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h
index 408783f532..1e3221cbec 100644
--- a/include/qemu/osdep.h
+++ b/include/qemu/osdep.h
@@ -158,6 +158,20 @@ extern int daemon(int, int);
/* Round number up to multiple */
#define QEMU_ALIGN_UP(n, m) QEMU_ALIGN_DOWN((n) + (m) - 1, (m))
+/* Check if n is a multiple of m */
+#define QEMU_IS_ALIGNED(n, m) (((n) % (m)) == 0)
+
+/* n-byte align pointer down */
+#define QEMU_ALIGN_PTR_DOWN(p, n) \
+ ((typeof(p))QEMU_ALIGN_DOWN((uintptr_t)(p), (n)))
+
+/* n-byte align pointer up */
+#define QEMU_ALIGN_PTR_UP(p, n) \
+ ((typeof(p))QEMU_ALIGN_UP((uintptr_t)(p), (n)))
+
+/* Check if pointer p is n-bytes aligned */
+#define QEMU_PTR_IS_ALIGNED(p, n) QEMU_IS_ALIGNED((uintptr_t)(p), (n))
+
#ifndef ROUND_UP
#define ROUND_UP(n,d) (((n) + (d) - 1) & -(d))
#endif
diff --git a/include/qom/cpu.h b/include/qom/cpu.h
index b7a10f791a..4349c465c5 100644
--- a/include/qom/cpu.h
+++ b/include/qom/cpu.h
@@ -238,6 +238,7 @@ struct kvm_run;
* @crash_occurred: Indicates the OS reported a crash (panic) for this CPU
* @tcg_exit_req: Set to force TCG to stop executing linked TBs for this
* CPU and return to its top level loop.
+ * @tb_flushed: Indicates the translation buffer has been flushed.
* @singlestep_enabled: Flags for single-stepping.
* @icount_extra: Instructions until next timer event.
* @icount_decr: Number of cycles left, with interrupt flag in high bit.
@@ -252,7 +253,6 @@ struct kvm_run;
* @as: Pointer to the first AddressSpace, for the convenience of targets which
* only have a single AddressSpace
* @env_ptr: Pointer to subclass-specific CPUArchState field.
- * @current_tb: Currently executing TB.
* @gdb_regs: Additional GDB registers.
* @gdb_num_regs: Number of total registers accessible to GDB.
* @gdb_num_g_regs: Number of registers in GDB 'g' packets.
@@ -289,6 +289,7 @@ struct CPUState {
bool stopped;
bool crash_occurred;
bool exit_request;
+ bool tb_flushed;
uint32_t interrupt_request;
int singlestep_enabled;
int64_t icount_extra;
@@ -303,7 +304,6 @@ struct CPUState {
MemoryRegion *memory;
void *env_ptr; /* CPUArchState */
- struct TranslationBlock *current_tb;
struct TranslationBlock *tb_jmp_cache[TB_JMP_CACHE_SIZE];
struct GDBRegisterState *gdb_regs;
int gdb_num_regs;
diff --git a/qom/cpu.c b/qom/cpu.c
index c9007d3d06..751e992de8 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -254,7 +254,6 @@ static void cpu_common_reset(CPUState *cpu)
}
cpu->interrupt_request = 0;
- cpu->current_tb = NULL;
cpu->halted = 0;
cpu->mem_io_pc = 0;
cpu->mem_io_vaddr = 0;
diff --git a/target-alpha/cpu.h b/target-alpha/cpu.h
index 420f2a53fe..b25d7d09d0 100644
--- a/target-alpha/cpu.h
+++ b/target-alpha/cpu.h
@@ -465,7 +465,7 @@ enum {
};
static inline void cpu_get_tb_cpu_state(CPUAlphaState *env, target_ulong *pc,
- target_ulong *cs_base, int *pflags)
+ target_ulong *cs_base, uint32_t *pflags)
{
int flags = 0;
diff --git a/target-alpha/translate.c b/target-alpha/translate.c
index 5b86992dd3..8c2183a418 100644
--- a/target-alpha/translate.c
+++ b/target-alpha/translate.c
@@ -460,12 +460,16 @@ static bool use_goto_tb(DisasContext *ctx, uint64_t dest)
|| ctx->singlestep_enabled || singlestep) {
return false;
}
+#ifndef CONFIG_USER_ONLY
/* If the destination is in the superpage, the page perms can't change. */
if (in_superpage(ctx, dest)) {
return true;
}
/* Check for the dest on the same page as the start of the TB. */
return ((ctx->tb->pc ^ dest) & TARGET_PAGE_MASK) == 0;
+#else
+ return true;
+#endif
}
static ExitStatus gen_bdirect(DisasContext *ctx, int ra, int32_t disp)
diff --git a/target-arm/cpu.h b/target-arm/cpu.h
index 066ff678dc..9deef86786 100644
--- a/target-arm/cpu.h
+++ b/target-arm/cpu.h
@@ -2117,7 +2117,7 @@ static inline bool arm_cpu_bswap_data(CPUARMState *env)
#endif
static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
if (is_a64(env)) {
*pc = env->pc;
diff --git a/target-arm/translate-a64.c b/target-arm/translate-a64.c
index 24f5e177dd..5526bbda2c 100644
--- a/target-arm/translate-a64.c
+++ b/target-arm/translate-a64.c
@@ -274,10 +274,12 @@ static inline bool use_goto_tb(DisasContext *s, int n, uint64_t dest)
return false;
}
+#ifndef CONFIG_USER_ONLY
/* Only link tbs from inside the same guest page */
if ((s->tb->pc & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) {
return false;
}
+#endif
return true;
}
diff --git a/target-arm/translate.c b/target-arm/translate.c
index 940ec8d981..a43b1f61cf 100644
--- a/target-arm/translate.c
+++ b/target-arm/translate.c
@@ -4049,15 +4049,22 @@ static int disas_vfp_insn(DisasContext *s, uint32_t insn)
return 0;
}
-static inline void gen_goto_tb(DisasContext *s, int n, target_ulong dest)
+static inline bool use_goto_tb(DisasContext *s, target_ulong dest)
{
- TranslationBlock *tb;
+#ifndef CONFIG_USER_ONLY
+ return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) ||
+ ((s->pc - 1) & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
- tb = s->tb;
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+static inline void gen_goto_tb(DisasContext *s, int n, target_ulong dest)
+{
+ if (use_goto_tb(s, dest)) {
tcg_gen_goto_tb(n);
gen_set_pc_im(s, dest);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)s->tb + n);
} else {
gen_set_pc_im(s, dest);
tcg_gen_exit_tb(0);
diff --git a/target-cris/cpu.h b/target-cris/cpu.h
index 415cf91436..a492fc687f 100644
--- a/target-cris/cpu.h
+++ b/target-cris/cpu.h
@@ -249,7 +249,7 @@ int cris_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
#include "exec/cpu-all.h"
static inline void cpu_get_tb_cpu_state(CPUCRISState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->pc;
*cs_base = 0;
diff --git a/target-cris/translate.c b/target-cris/translate.c
index a73176c118..f28b1999a7 100644
--- a/target-cris/translate.c
+++ b/target-cris/translate.c
@@ -520,14 +520,22 @@ static void t_gen_cc_jmp(TCGv pc_true, TCGv pc_false)
gen_set_label(l1);
}
+static inline bool use_goto_tb(DisasContext *dc, target_ulong dest)
+{
+#ifndef CONFIG_USER_ONLY
+ return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) ||
+ (dc->ppc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
+
static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
{
- TranslationBlock *tb;
- tb = dc->tb;
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+ if (use_goto_tb(dc, dest)) {
tcg_gen_goto_tb(n);
tcg_gen_movi_tl(env_pc, dest);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)dc->tb + n);
} else {
tcg_gen_movi_tl(env_pc, dest);
tcg_gen_exit_tb(0);
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index 732eb6d7ec..444fda9ce6 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -1269,7 +1269,7 @@ void tcg_x86_init(void);
#include "exec/exec-all.h"
static inline void cpu_get_tb_cpu_state(CPUX86State *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*cs_base = env->segs[R_CS].base;
*pc = *cs_base + env->eip;
diff --git a/target-i386/translate.c b/target-i386/translate.c
index 1a1214dcb1..868c26244b 100644
--- a/target-i386/translate.c
+++ b/target-i386/translate.c
@@ -2085,20 +2085,25 @@ static inline int insn_const_size(TCGMemOp ot)
}
}
+static inline bool use_goto_tb(DisasContext *s, target_ulong pc)
+{
+#ifndef CONFIG_USER_ONLY
+ return (pc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) ||
+ (pc & TARGET_PAGE_MASK) == (s->pc_start & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
+
static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip)
{
- TranslationBlock *tb;
- target_ulong pc;
-
- pc = s->cs_base + eip;
- tb = s->tb;
- /* NOTE: we handle the case where the TB spans two pages here */
- if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) ||
- (pc & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK)) {
+ target_ulong pc = s->cs_base + eip;
+
+ if (use_goto_tb(s, pc)) {
/* jump to same page: we can use a direct jump */
tcg_gen_goto_tb(tb_num);
gen_jmp_im(eip);
- tcg_gen_exit_tb((uintptr_t)tb + tb_num);
+ tcg_gen_exit_tb((uintptr_t)s->tb + tb_num);
} else {
/* jump to another page: currently not optimized */
gen_jmp_im(eip);
@@ -8178,7 +8183,7 @@ void gen_intermediate_code(CPUX86State *env, TranslationBlock *tb)
CPUState *cs = CPU(cpu);
DisasContext dc1, *dc = &dc1;
target_ulong pc_ptr;
- uint64_t flags;
+ uint32_t flags;
target_ulong pc_start;
target_ulong cs_base;
int num_insns;
diff --git a/target-lm32/cpu.h b/target-lm32/cpu.h
index f220fc0bb9..6a0d297b30 100644
--- a/target-lm32/cpu.h
+++ b/target-lm32/cpu.h
@@ -226,7 +226,7 @@ int lm32_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
#include "exec/cpu-all.h"
static inline void cpu_get_tb_cpu_state(CPULM32State *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->pc;
*cs_base = 0;
diff --git a/target-lm32/translate.c b/target-lm32/translate.c
index 256a51f849..dd972f5b8c 100644
--- a/target-lm32/translate.c
+++ b/target-lm32/translate.c
@@ -133,16 +133,25 @@ static inline void t_gen_illegal_insn(DisasContext *dc)
gen_helper_ill(cpu_env);
}
-static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
+static inline bool use_goto_tb(DisasContext *dc, target_ulong dest)
{
- TranslationBlock *tb;
+ if (unlikely(dc->singlestep_enabled)) {
+ return false;
+ }
+
+#ifndef CONFIG_USER_ONLY
+ return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
- tb = dc->tb;
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
- likely(!dc->singlestep_enabled)) {
+static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
+{
+ if (use_goto_tb(dc, dest)) {
tcg_gen_goto_tb(n);
tcg_gen_movi_tl(cpu_pc, dest);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)dc->tb + n);
} else {
tcg_gen_movi_tl(cpu_pc, dest);
if (dc->singlestep_enabled) {
diff --git a/target-m68k/cpu.h b/target-m68k/cpu.h
index 48b4c872fc..d2f467ca9b 100644
--- a/target-m68k/cpu.h
+++ b/target-m68k/cpu.h
@@ -230,7 +230,7 @@ int m68k_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
#include "exec/cpu-all.h"
static inline void cpu_get_tb_cpu_state(CPUM68KState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->pc;
*cs_base = 0;
diff --git a/target-m68k/translate.c b/target-m68k/translate.c
index 7560c3a808..e46356e44c 100644
--- a/target-m68k/translate.c
+++ b/target-m68k/translate.c
@@ -852,19 +852,25 @@ static inline void gen_addr_fault(DisasContext *s)
} \
} while (0)
+static inline bool use_goto_tb(DisasContext *s, uint32_t dest)
+{
+#ifndef CONFIG_USER_ONLY
+ return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) ||
+ (s->insn_pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
+
/* Generate a jump to an immediate address. */
static void gen_jmp_tb(DisasContext *s, int n, uint32_t dest)
{
- TranslationBlock *tb;
-
- tb = s->tb;
if (unlikely(s->singlestep_enabled)) {
gen_exception(s, dest, EXCP_DEBUG);
- } else if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) ||
- (s->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+ } else if (use_goto_tb(s, dest)) {
tcg_gen_goto_tb(n);
tcg_gen_movi_i32(QREG_PC, dest);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)s->tb + n);
} else {
gen_jmp_im(s, dest);
tcg_gen_exit_tb(0);
diff --git a/target-microblaze/cpu.h b/target-microblaze/cpu.h
index 2f7335eaa7..bf74e2c032 100644
--- a/target-microblaze/cpu.h
+++ b/target-microblaze/cpu.h
@@ -322,7 +322,7 @@ int mb_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
#include "exec/cpu-all.h"
static inline void cpu_get_tb_cpu_state(CPUMBState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->sregs[SR_PC];
*cs_base = 0;
diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c
index f944965a14..a7a8ac8f99 100644
--- a/target-microblaze/translate.c
+++ b/target-microblaze/translate.c
@@ -124,14 +124,21 @@ static inline void t_gen_raise_exception(DisasContext *dc, uint32_t index)
dc->is_jmp = DISAS_UPDATE;
}
+static inline bool use_goto_tb(DisasContext *dc, target_ulong dest)
+{
+#ifndef CONFIG_USER_ONLY
+ return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
+
static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
{
- TranslationBlock *tb;
- tb = dc->tb;
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+ if (use_goto_tb(dc, dest)) {
tcg_gen_goto_tb(n);
tcg_gen_movi_tl(cpu_SR[SR_PC], dest);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)dc->tb + n);
} else {
tcg_gen_movi_tl(cpu_SR[SR_PC], dest);
tcg_gen_exit_tb(0);
diff --git a/target-mips/cpu.h b/target-mips/cpu.h
index 866924d188..53e826223f 100644
--- a/target-mips/cpu.h
+++ b/target-mips/cpu.h
@@ -839,7 +839,7 @@ static inline void restore_pamask(CPUMIPSState *env)
}
static inline void cpu_get_tb_cpu_state(CPUMIPSState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->active_tc.PC;
*cs_base = 0;
diff --git a/target-mips/translate.c b/target-mips/translate.c
index a3a05ec66d..ddfb9244d7 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -4191,15 +4191,25 @@ static void gen_trap (DisasContext *ctx, uint32_t opc,
tcg_temp_free(t1);
}
+static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest)
+{
+ if (unlikely(ctx->singlestep_enabled)) {
+ return false;
+ }
+
+#ifndef CONFIG_USER_ONLY
+ return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
+
static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
{
- TranslationBlock *tb;
- tb = ctx->tb;
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
- likely(!ctx->singlestep_enabled)) {
+ if (use_goto_tb(ctx, dest)) {
tcg_gen_goto_tb(n);
gen_save_pc(dest);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)ctx->tb + n);
} else {
gen_save_pc(dest);
if (ctx->singlestep_enabled) {
diff --git a/target-moxie/cpu.h b/target-moxie/cpu.h
index 4ee207796c..1b46e52c37 100644
--- a/target-moxie/cpu.h
+++ b/target-moxie/cpu.h
@@ -132,7 +132,7 @@ static inline int cpu_mmu_index(CPUMoxieState *env, bool ifetch)
#include "exec/exec-all.h"
static inline void cpu_get_tb_cpu_state(CPUMoxieState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->pc;
*cs_base = 0;
diff --git a/target-moxie/translate.c b/target-moxie/translate.c
index a437e2ab60..58200c25d3 100644
--- a/target-moxie/translate.c
+++ b/target-moxie/translate.c
@@ -121,17 +121,26 @@ void moxie_translate_init(void)
done_init = 1;
}
+static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest)
+{
+ if (unlikely(ctx->singlestep_enabled)) {
+ return false;
+ }
+
+#ifndef CONFIG_USER_ONLY
+ return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
+
static inline void gen_goto_tb(CPUMoxieState *env, DisasContext *ctx,
int n, target_ulong dest)
{
- TranslationBlock *tb;
- tb = ctx->tb;
-
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
- !ctx->singlestep_enabled) {
+ if (use_goto_tb(ctx, dest)) {
tcg_gen_goto_tb(n);
tcg_gen_movi_i32(cpu_pc, dest);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)ctx->tb + n);
} else {
tcg_gen_movi_i32(cpu_pc, dest);
if (ctx->singlestep_enabled) {
diff --git a/target-openrisc/cpu.h b/target-openrisc/cpu.h
index 4b63f25804..ed818af0cf 100644
--- a/target-openrisc/cpu.h
+++ b/target-openrisc/cpu.h
@@ -392,7 +392,7 @@ int cpu_openrisc_get_phys_data(OpenRISCCPU *cpu,
static inline void cpu_get_tb_cpu_state(CPUOpenRISCState *env,
target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->pc;
*cs_base = 0;
diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c
index 5d0ab442a8..d4f1f260e4 100644
--- a/target-openrisc/translate.c
+++ b/target-openrisc/translate.c
@@ -190,15 +190,25 @@ static void check_ov64s(DisasContext *dc)
}
#endif*/
+static inline bool use_goto_tb(DisasContext *dc, target_ulong dest)
+{
+ if (unlikely(dc->singlestep_enabled)) {
+ return false;
+ }
+
+#ifndef CONFIG_USER_ONLY
+ return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
+
static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
{
- TranslationBlock *tb;
- tb = dc->tb;
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
- likely(!dc->singlestep_enabled)) {
+ if (use_goto_tb(dc, dest)) {
tcg_gen_movi_tl(cpu_pc, dest);
tcg_gen_goto_tb(n);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)dc->tb + n);
} else {
tcg_gen_movi_tl(cpu_pc, dest);
if (dc->singlestep_enabled) {
diff --git a/target-ppc/cpu.h b/target-ppc/cpu.h
index 5282533b38..508f03b74d 100644
--- a/target-ppc/cpu.h
+++ b/target-ppc/cpu.h
@@ -2303,7 +2303,7 @@ static inline void cpu_write_xer(CPUPPCState *env, target_ulong xer)
}
static inline void cpu_get_tb_cpu_state(CPUPPCState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->nip;
*cs_base = 0;
diff --git a/target-ppc/translate.c b/target-ppc/translate.c
index b3860ecdea..d485d7c7cb 100644
--- a/target-ppc/translate.c
+++ b/target-ppc/translate.c
@@ -3822,19 +3822,29 @@ static inline void gen_update_cfar(DisasContext *ctx, target_ulong nip)
#endif
}
+static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest)
+{
+ if (unlikely(ctx->singlestep_enabled)) {
+ return false;
+ }
+
+#ifndef CONFIG_USER_ONLY
+ return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
+
/*** Branch ***/
static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
{
- TranslationBlock *tb;
- tb = ctx->tb;
if (NARROW_MODE(ctx)) {
dest = (uint32_t) dest;
}
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
- likely(!ctx->singlestep_enabled)) {
+ if (use_goto_tb(ctx, dest)) {
tcg_gen_goto_tb(n);
tcg_gen_movi_tl(cpu_nip, dest & ~3);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)ctx->tb + n);
} else {
tcg_gen_movi_tl(cpu_nip, dest & ~3);
if (unlikely(ctx->singlestep_enabled)) {
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 6d97c089a4..07f76ad884 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -338,7 +338,7 @@ static inline uint64_t cpu_mmu_idx_to_asc(int mmu_idx)
}
static inline void cpu_get_tb_cpu_state(CPUS390XState* env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->psw.addr;
*cs_base = 0;
diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index c871ef2bb3..e99eb5cb01 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -608,12 +608,17 @@ static void gen_op_calc_cc(DisasContext *s)
static int use_goto_tb(DisasContext *s, uint64_t dest)
{
- /* NOTE: we handle the case where the TB spans two pages here */
- return (((dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK)
- || (dest & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK))
- && !s->singlestep_enabled
- && !(s->tb->cflags & CF_LAST_IO)
- && !(s->tb->flags & FLAG_MASK_PER));
+ if (unlikely(s->singlestep_enabled) ||
+ (s->tb->cflags & CF_LAST_IO) ||
+ (s->tb->flags & FLAG_MASK_PER)) {
+ return false;
+ }
+#ifndef CONFIG_USER_ONLY
+ return (dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) ||
+ (dest & TARGET_PAGE_MASK) == (s->pc & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
}
static void account_noninline_branch(DisasContext *s, int cc_op)
diff --git a/target-sh4/cpu.h b/target-sh4/cpu.h
index 3b23e967bb..10c0191795 100644
--- a/target-sh4/cpu.h
+++ b/target-sh4/cpu.h
@@ -347,7 +347,7 @@ static inline void cpu_write_sr(CPUSH4State *env, target_ulong sr)
}
static inline void cpu_get_tb_cpu_state(CPUSH4State *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->pc;
*cs_base = 0;
diff --git a/target-sh4/translate.c b/target-sh4/translate.c
index 7c189680a7..53f782c054 100644
--- a/target-sh4/translate.c
+++ b/target-sh4/translate.c
@@ -205,17 +205,26 @@ static void gen_write_sr(TCGv src)
tcg_gen_andi_i32(cpu_sr_t, cpu_sr_t, 1);
}
-static void gen_goto_tb(DisasContext * ctx, int n, target_ulong dest)
+static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest)
{
- TranslationBlock *tb;
- tb = ctx->tb;
+ if (unlikely(ctx->singlestep_enabled)) {
+ return false;
+ }
+
+#ifndef CONFIG_USER_ONLY
+ return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
- !ctx->singlestep_enabled) {
+static void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
+{
+ if (use_goto_tb(ctx, dest)) {
/* Use a direct jump if in same page and singlestep not enabled */
tcg_gen_goto_tb(n);
tcg_gen_movi_i32(cpu_pc, dest);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)ctx->tb + n);
} else {
tcg_gen_movi_i32(cpu_pc, dest);
if (ctx->singlestep_enabled)
diff --git a/target-sparc/cpu.h b/target-sparc/cpu.h
index dc46122758..59ec7cafbe 100644
--- a/target-sparc/cpu.h
+++ b/target-sparc/cpu.h
@@ -688,7 +688,7 @@ trap_state* cpu_tsptr(CPUSPARCState* env);
#define TB_FLAG_AM_ENABLED (1 << 5)
static inline void cpu_get_tb_cpu_state(CPUSPARCState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->pc;
*cs_base = env->npc;
diff --git a/target-sparc/translate.c b/target-sparc/translate.c
index 7998ff57bf..d154e3f7b6 100644
--- a/target-sparc/translate.c
+++ b/target-sparc/translate.c
@@ -303,20 +303,30 @@ static inline TCGv gen_dest_gpr(DisasContext *dc, int reg)
}
}
+static inline bool use_goto_tb(DisasContext *s, target_ulong pc,
+ target_ulong npc)
+{
+ if (unlikely(s->singlestep)) {
+ return false;
+ }
+
+#ifndef CONFIG_USER_ONLY
+ return (pc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) &&
+ (npc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
+
static inline void gen_goto_tb(DisasContext *s, int tb_num,
target_ulong pc, target_ulong npc)
{
- TranslationBlock *tb;
-
- tb = s->tb;
- if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) &&
- (npc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) &&
- !s->singlestep) {
+ if (use_goto_tb(s, pc, npc)) {
/* jump to same page: we can use a direct jump */
tcg_gen_goto_tb(tb_num);
tcg_gen_movi_tl(cpu_pc, pc);
tcg_gen_movi_tl(cpu_npc, npc);
- tcg_gen_exit_tb((uintptr_t)tb + tb_num);
+ tcg_gen_exit_tb((uintptr_t)s->tb + tb_num);
} else {
/* jump to another page: currently not optimized */
tcg_gen_movi_tl(cpu_pc, pc);
diff --git a/target-tilegx/cpu.h b/target-tilegx/cpu.h
index 022cad186a..c9dda127c9 100644
--- a/target-tilegx/cpu.h
+++ b/target-tilegx/cpu.h
@@ -169,7 +169,7 @@ TileGXCPU *cpu_tilegx_init(const char *cpu_model);
#define cpu_signal_handler cpu_tilegx_signal_handler
static inline void cpu_get_tb_cpu_state(CPUTLGState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->pc;
*cs_base = 0;
diff --git a/target-tricore/cpu.h b/target-tricore/cpu.h
index 90045a93d2..eaebdd28e6 100644
--- a/target-tricore/cpu.h
+++ b/target-tricore/cpu.h
@@ -377,7 +377,7 @@ void tricore_tcg_init(void);
int cpu_tricore_signal_handler(int host_signum, void *pinfo, void *puc);
static inline void cpu_get_tb_cpu_state(CPUTriCoreState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->PC;
*cs_base = 0;
diff --git a/target-tricore/translate.c b/target-tricore/translate.c
index 912bf226be..0237e7bea8 100644
--- a/target-tricore/translate.c
+++ b/target-tricore/translate.c
@@ -3236,15 +3236,25 @@ static inline void gen_save_pc(target_ulong pc)
tcg_gen_movi_tl(cpu_PC, pc);
}
+static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest)
+{
+ if (unlikely(ctx->singlestep_enabled)) {
+ return false;
+ }
+
+#ifndef CONFIG_USER_ONLY
+ return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
+
static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
{
- TranslationBlock *tb;
- tb = ctx->tb;
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
- likely(!ctx->singlestep_enabled)) {
+ if (use_goto_tb(ctx, dest)) {
tcg_gen_goto_tb(n);
gen_save_pc(dest);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)ctx->tb + n);
} else {
gen_save_pc(dest);
if (ctx->singlestep_enabled) {
diff --git a/target-unicore32/cpu.h b/target-unicore32/cpu.h
index 9c1fbf9b77..e64cb7ecdd 100644
--- a/target-unicore32/cpu.h
+++ b/target-unicore32/cpu.h
@@ -144,7 +144,7 @@ UniCore32CPU *uc32_cpu_init(const char *cpu_model);
#define cpu_init(cpu_model) CPU(uc32_cpu_init(cpu_model))
static inline void cpu_get_tb_cpu_state(CPUUniCore32State *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
*pc = env->regs[31];
*cs_base = 0;
diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c
index 39af3af05f..307f7b2059 100644
--- a/target-unicore32/translate.c
+++ b/target-unicore32/translate.c
@@ -1089,15 +1089,21 @@ static void disas_ucf64_insn(CPUUniCore32State *env, DisasContext *s, uint32_t i
}
}
-static inline void gen_goto_tb(DisasContext *s, int n, uint32_t dest)
+static inline bool use_goto_tb(DisasContext *s, uint32_t dest)
{
- TranslationBlock *tb;
+#ifndef CONFIG_USER_ONLY
+ return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+ return true;
+#endif
+}
- tb = s->tb;
- if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+static inline void gen_goto_tb(DisasContext *s, int n, uint32_t dest)
+{
+ if (use_goto_tb(s, dest)) {
tcg_gen_goto_tb(n);
gen_set_pc_im(dest);
- tcg_gen_exit_tb((uintptr_t)tb + n);
+ tcg_gen_exit_tb((uintptr_t)s->tb + n);
} else {
gen_set_pc_im(dest);
tcg_gen_exit_tb(0);
diff --git a/target-xtensa/cpu.h b/target-xtensa/cpu.h
index d0bd9dada8..7bfc9c841d 100644
--- a/target-xtensa/cpu.h
+++ b/target-xtensa/cpu.h
@@ -507,7 +507,7 @@ static inline int cpu_mmu_index(CPUXtensaState *env, bool ifetch)
#define XTENSA_TBFLAG_WINDOW_SHIFT 15
static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc,
- target_ulong *cs_base, int *flags)
+ target_ulong *cs_base, uint32_t *flags)
{
CPUState *cs = CPU(xtensa_env_get_cpu(env));
diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c
index 9894488469..9eac56e2a5 100644
--- a/target-xtensa/translate.c
+++ b/target-xtensa/translate.c
@@ -418,9 +418,11 @@ static void gen_jump(DisasContext *dc, TCGv dest)
static void gen_jumpi(DisasContext *dc, uint32_t dest, int slot)
{
TCGv_i32 tmp = tcg_const_i32(dest);
+#ifndef CONFIG_USER_ONLY
if (((dc->tb->pc ^ dest) & TARGET_PAGE_MASK) != 0) {
slot = -1;
}
+#endif
gen_jump_slot(dc, tmp, slot);
tcg_temp_free(tmp);
}
@@ -446,9 +448,11 @@ static void gen_callw(DisasContext *dc, int callinc, TCGv_i32 dest)
static void gen_callwi(DisasContext *dc, int callinc, uint32_t dest, int slot)
{
TCGv_i32 tmp = tcg_const_i32(dest);
+#ifndef CONFIG_USER_ONLY
if (((dc->tb->pc ^ dest) & TARGET_PAGE_MASK) != 0) {
slot = -1;
}
+#endif
gen_callw_slot(dc, callinc, tmp, slot);
tcg_temp_free(tmp);
}
diff --git a/tcg/aarch64/tcg-target.inc.c b/tcg/aarch64/tcg-target.inc.c
index a8fb4420de..1447f7c216 100644
--- a/tcg/aarch64/tcg-target.inc.c
+++ b/tcg/aarch64/tcg-target.inc.c
@@ -73,6 +73,18 @@ static inline void reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
*code_ptr = deposit32(*code_ptr, 0, 26, offset);
}
+static inline void reloc_pc26_atomic(tcg_insn_unit *code_ptr,
+ tcg_insn_unit *target)
+{
+ ptrdiff_t offset = target - code_ptr;
+ tcg_insn_unit insn;
+ tcg_debug_assert(offset == sextract64(offset, 0, 26));
+ /* read instruction, mask away previous PC_REL26 parameter contents,
+ set the proper offset, then write back the instruction. */
+ insn = atomic_read(code_ptr);
+ atomic_set(code_ptr, deposit32(insn, 0, 26, offset));
+}
+
static inline void reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
{
ptrdiff_t offset = target - code_ptr;
@@ -835,7 +847,7 @@ void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
tcg_insn_unit *code_ptr = (tcg_insn_unit *)jmp_addr;
tcg_insn_unit *target = (tcg_insn_unit *)addr;
- reloc_pc26(code_ptr, target);
+ reloc_pc26_atomic(code_ptr, target);
flush_icache_range(jmp_addr, jmp_addr + 4);
}
@@ -1294,12 +1306,13 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
#ifndef USE_DIRECT_JUMP
#error "USE_DIRECT_JUMP required for aarch64"
#endif
- tcg_debug_assert(s->tb_jmp_offset != NULL); /* consistency for USE_DIRECT_JUMP */
- s->tb_jmp_offset[a0] = tcg_current_code_size(s);
+ /* consistency for USE_DIRECT_JUMP */
+ tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
+ s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
/* actual branch destination will be patched by
aarch64_tb_set_jmp_target later, beware retranslation. */
tcg_out_goto_noaddr(s);
- s->tb_next_offset[a0] = tcg_current_code_size(s);
+ s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s);
break;
case INDEX_op_br:
diff --git a/tcg/arm/tcg-target.inc.c b/tcg/arm/tcg-target.inc.c
index 2b7fbddbf0..f9f54c64c6 100644
--- a/tcg/arm/tcg-target.inc.c
+++ b/tcg/arm/tcg-target.inc.c
@@ -121,6 +121,14 @@ static inline void reloc_pc24(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
*code_ptr = (*code_ptr & ~0xffffff) | (offset & 0xffffff);
}
+static inline void reloc_pc24_atomic(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+{
+ ptrdiff_t offset = (tcg_ptr_byte_diff(target, code_ptr) - 8) >> 2;
+ tcg_insn_unit insn = atomic_read(code_ptr);
+ tcg_debug_assert(offset == sextract32(offset, 0, 24));
+ atomic_set(code_ptr, deposit32(insn, 0, 24, offset));
+}
+
static void patch_reloc(tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
@@ -1038,6 +1046,16 @@ static void tcg_out_call(TCGContext *s, tcg_insn_unit *addr)
}
}
+void arm_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
+{
+ tcg_insn_unit *code_ptr = (tcg_insn_unit *)jmp_addr;
+ tcg_insn_unit *target = (tcg_insn_unit *)addr;
+
+ /* we could use a ldr pc, [pc, #-4] kind of branch and avoid the flush */
+ reloc_pc24_atomic(code_ptr, target);
+ flush_icache_range(jmp_addr, jmp_addr + 4);
+}
+
static inline void tcg_out_goto_label(TCGContext *s, int cond, TCGLabel *l)
{
if (l->has_value) {
@@ -1647,17 +1665,17 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
tcg_out_goto(s, COND_AL, tb_ret_addr);
break;
case INDEX_op_goto_tb:
- if (s->tb_jmp_offset) {
+ if (s->tb_jmp_insn_offset) {
/* Direct jump method */
- s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
+ s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
tcg_out_b_noaddr(s, COND_AL);
} else {
/* Indirect jump method */
- intptr_t ptr = (intptr_t)(s->tb_next + args[0]);
+ intptr_t ptr = (intptr_t)(s->tb_jmp_target_addr + args[0]);
tcg_out_movi32(s, COND_AL, TCG_REG_R0, ptr & ~0xfff);
tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_R0, ptr & 0xfff);
}
- s->tb_next_offset[args[0]] = tcg_current_code_size(s);
+ s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
break;
case INDEX_op_br:
tcg_out_goto_label(s, COND_AL, arg_label(args[0]));
diff --git a/tcg/i386/tcg-target.inc.c b/tcg/i386/tcg-target.inc.c
index 007407c3fc..317484cb5d 100644
--- a/tcg/i386/tcg-target.inc.c
+++ b/tcg/i386/tcg-target.inc.c
@@ -1123,6 +1123,21 @@ static void tcg_out_jmp(TCGContext *s, tcg_insn_unit *dest)
tcg_out_branch(s, 0, dest);
}
+static void tcg_out_nopn(TCGContext *s, int n)
+{
+ int i;
+ /* Emit 1 or 2 operand size prefixes for the standard one byte nop,
+ * "xchg %eax,%eax", forming "xchg %ax,%ax". All cores accept the
+ * duplicate prefix, and all of the interesting recent cores can
+ * decode and discard the duplicates in a single cycle.
+ */
+ tcg_debug_assert(n >= 1);
+ for (i = 1; i < n; ++i) {
+ tcg_out8(s, 0x66);
+ }
+ tcg_out8(s, 0x90);
+}
+
#if defined(CONFIG_SOFTMMU)
/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
* int mmu_idx, uintptr_t ra)
@@ -1775,17 +1790,25 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
tcg_out_jmp(s, tb_ret_addr);
break;
case INDEX_op_goto_tb:
- if (s->tb_jmp_offset) {
+ if (s->tb_jmp_insn_offset) {
/* direct jump method */
+ int gap;
+ /* jump displacement must be aligned for atomic patching;
+ * see if we need to add extra nops before jump
+ */
+ gap = tcg_pcrel_diff(s, QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4));
+ if (gap != 1) {
+ tcg_out_nopn(s, gap - 1);
+ }
tcg_out8(s, OPC_JMP_long); /* jmp im */
- s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
+ s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
tcg_out32(s, 0);
} else {
/* indirect jump method */
tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
- (intptr_t)(s->tb_next + args[0]));
+ (intptr_t)(s->tb_jmp_target_addr + args[0]));
}
- s->tb_next_offset[args[0]] = tcg_current_code_size(s);
+ s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
break;
case INDEX_op_br:
tcg_out_jxx(s, JCC_JMP, arg_label(args[0]), 0);
diff --git a/tcg/ia64/tcg-target.inc.c b/tcg/ia64/tcg-target.inc.c
index 7557e6a9d4..395223e340 100644
--- a/tcg/ia64/tcg-target.inc.c
+++ b/tcg/ia64/tcg-target.inc.c
@@ -881,13 +881,13 @@ static void tcg_out_exit_tb(TCGContext *s, tcg_target_long arg)
static inline void tcg_out_goto_tb(TCGContext *s, TCGArg arg)
{
- if (s->tb_jmp_offset) {
+ if (s->tb_jmp_insn_offset) {
/* direct jump method */
tcg_abort();
} else {
/* indirect jump method */
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2,
- (tcg_target_long)(s->tb_next + arg));
+ (tcg_target_long)(s->tb_jmp_target_addr + arg));
tcg_out_bundle(s, MmI,
tcg_opc_m1 (TCG_REG_P0, OPC_LD8_M1,
TCG_REG_R2, TCG_REG_R2),
@@ -900,7 +900,7 @@ static inline void tcg_out_goto_tb(TCGContext *s, TCGArg arg)
tcg_opc_b4 (TCG_REG_P0, OPC_BR_SPTK_MANY_B4,
TCG_REG_B6));
}
- s->tb_next_offset[arg] = tcg_current_code_size(s);
+ s->tb_jmp_reset_offset[arg] = tcg_current_code_size(s);
}
static inline void tcg_out_jmp(TCGContext *s, TCGArg addr)
diff --git a/tcg/mips/tcg-target.inc.c b/tcg/mips/tcg-target.inc.c
index aaf881cfd0..50e98ea63a 100644
--- a/tcg/mips/tcg-target.inc.c
+++ b/tcg/mips/tcg-target.inc.c
@@ -1397,19 +1397,19 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
}
break;
case INDEX_op_goto_tb:
- if (s->tb_jmp_offset) {
+ if (s->tb_jmp_insn_offset) {
/* direct jump method */
- s->tb_jmp_offset[a0] = tcg_current_code_size(s);
+ s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
/* Avoid clobbering the address during retranslation. */
tcg_out32(s, OPC_J | (*(uint32_t *)s->code_ptr & 0x3ffffff));
} else {
/* indirect jump method */
tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
- (uintptr_t)(s->tb_next + a0));
+ (uintptr_t)(s->tb_jmp_target_addr + a0));
tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
}
tcg_out_nop(s);
- s->tb_next_offset[a0] = tcg_current_code_size(s);
+ s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s);
break;
case INDEX_op_br:
tcg_out_brcond(s, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO,
@@ -1885,7 +1885,6 @@ static void tcg_target_init(TCGContext *s)
void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
{
- uint32_t *ptr = (uint32_t *)jmp_addr;
- *ptr = deposit32(*ptr, 0, 26, addr >> 2);
+ atomic_set((uint32_t *)jmp_addr, deposit32(OPC_J, 0, 26, addr >> 2));
flush_icache_range(jmp_addr, jmp_addr + 4);
}
diff --git a/tcg/ppc/tcg-target.inc.c b/tcg/ppc/tcg-target.inc.c
index 00bb90fc25..da100528ab 100644
--- a/tcg/ppc/tcg-target.inc.c
+++ b/tcg/ppc/tcg-target.inc.c
@@ -1237,6 +1237,7 @@ static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args,
tcg_out_bc(s, BC | BI(7, CR_EQ) | BO_COND_TRUE, arg_label(args[5]));
}
+#ifdef __powerpc64__
void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
{
tcg_insn_unit i1, i2;
@@ -1265,11 +1266,18 @@ void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
pair = (uint64_t)i2 << 32 | i1;
#endif
- /* ??? __atomic_store_8, presuming there's some way to do that
- for 32-bit, otherwise this is good enough for 64-bit. */
- *(uint64_t *)jmp_addr = pair;
+ atomic_set((uint64_t *)jmp_addr, pair);
flush_icache_range(jmp_addr, jmp_addr + 8);
}
+#else
+void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
+{
+ intptr_t diff = addr - jmp_addr;
+ tcg_debug_assert(in_range_b(diff));
+ atomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc));
+ flush_icache_range(jmp_addr, jmp_addr + 4);
+}
+#endif
static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
{
@@ -1894,17 +1902,23 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
tcg_out_b(s, 0, tb_ret_addr);
break;
case INDEX_op_goto_tb:
- tcg_debug_assert(s->tb_jmp_offset);
- /* Direct jump. Ensure the next insns are 8-byte aligned. */
+ tcg_debug_assert(s->tb_jmp_insn_offset);
+ /* Direct jump. */
+#ifdef __powerpc64__
+ /* Ensure the next insns are 8-byte aligned. */
if ((uintptr_t)s->code_ptr & 7) {
tcg_out32(s, NOP);
}
- s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
+ s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
/* To be replaced by either a branch+nop or a load into TMP1. */
s->code_ptr += 2;
tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR);
tcg_out32(s, BCCTR | BO_ALWAYS);
- s->tb_next_offset[args[0]] = tcg_current_code_size(s);
+#else
+ /* To be replaced by a branch. */
+ s->code_ptr++;
+#endif
+ s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
break;
case INDEX_op_br:
{
diff --git a/tcg/s390/tcg-target.inc.c b/tcg/s390/tcg-target.inc.c
index 5805532398..e0a60e618c 100644
--- a/tcg/s390/tcg-target.inc.c
+++ b/tcg/s390/tcg-target.inc.c
@@ -219,6 +219,8 @@ typedef enum S390Opcode {
RX_ST = 0x50,
RX_STC = 0x42,
RX_STH = 0x40,
+
+ NOP = 0x0707,
} S390Opcode;
#ifdef CONFIG_DEBUG_TCG
@@ -1715,17 +1717,24 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
break;
case INDEX_op_goto_tb:
- if (s->tb_jmp_offset) {
+ if (s->tb_jmp_insn_offset) {
+ /* branch displacement must be aligned for atomic patching;
+ * see if we need to add extra nop before branch
+ */
+ if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
+ tcg_out16(s, NOP);
+ }
tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
- s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
+ s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
s->code_ptr += 2;
} else {
- /* load address stored at s->tb_next + args[0] */
- tcg_out_ld_abs(s, TCG_TYPE_PTR, TCG_TMP0, s->tb_next + args[0]);
+ /* load address stored at s->tb_jmp_target_addr + args[0] */
+ tcg_out_ld_abs(s, TCG_TYPE_PTR, TCG_TMP0,
+ s->tb_jmp_target_addr + args[0]);
/* and go there */
tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_TMP0);
}
- s->tb_next_offset[args[0]] = tcg_current_code_size(s);
+ s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
break;
OP_32_64(ld8u):
diff --git a/tcg/sparc/tcg-target.inc.c b/tcg/sparc/tcg-target.inc.c
index d641cfd8c5..9938a5085e 100644
--- a/tcg/sparc/tcg-target.inc.c
+++ b/tcg/sparc/tcg-target.inc.c
@@ -1229,18 +1229,19 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
}
break;
case INDEX_op_goto_tb:
- if (s->tb_jmp_offset) {
+ if (s->tb_jmp_insn_offset) {
/* direct jump method */
- s->tb_jmp_offset[a0] = tcg_current_code_size(s);
+ s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
/* Make sure to preserve links during retranslation. */
tcg_out32(s, CALL | (*s->code_ptr & ~INSN_OP(-1)));
} else {
/* indirect jump method */
- tcg_out_ld_ptr(s, TCG_REG_T1, (uintptr_t)(s->tb_next + a0));
+ tcg_out_ld_ptr(s, TCG_REG_T1,
+ (uintptr_t)(s->tb_jmp_target_addr + a0));
tcg_out_arithi(s, TCG_REG_G0, TCG_REG_T1, 0, JMPL);
}
tcg_out_nop(s);
- s->tb_next_offset[a0] = tcg_current_code_size(s);
+ s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s);
break;
case INDEX_op_br:
tcg_out_bpcc(s, COND_A, BPCC_PT, arg_label(a0));
@@ -1647,6 +1648,6 @@ void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
the code_gen_buffer can't be larger than 2GB. */
tcg_debug_assert(disp == (int32_t)disp);
- *ptr = CALL | (uint32_t)disp >> 2;
+ atomic_set(ptr, deposit32(CALL, 0, 30, disp >> 2));
flush_icache_range(jmp_addr, jmp_addr + 4);
}
diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h
index c446d3dc72..f217e80747 100644
--- a/tcg/tcg-op.h
+++ b/tcg/tcg-op.h
@@ -753,6 +753,19 @@ static inline void tcg_gen_exit_tb(uintptr_t val)
tcg_gen_op1i(INDEX_op_exit_tb, val);
}
+/**
+ * tcg_gen_goto_tb() - output goto_tb TCG operation
+ * @idx: Direct jump slot index (0 or 1)
+ *
+ * See tcg/README for more info about this TCG operation.
+ *
+ * NOTE: In softmmu emulation, direct jumps with goto_tb are only safe within
+ * the pages this TB resides in because we don't take care of direct jumps when
+ * address mapping changes, e.g. in tlb_flush(). In user mode, there's only a
+ * static address translation, so the destination address is always valid, TBs
+ * are always invalidated properly, and direct jumps are reset when mapping
+ * changes.
+ */
void tcg_gen_goto_tb(unsigned idx);
#if TARGET_LONG_BITS == 32
diff --git a/tcg/tcg.h b/tcg/tcg.h
index 01dfebd9a2..a013d77a33 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -510,9 +510,9 @@ struct TCGContext {
/* goto_tb support */
tcg_insn_unit *code_buf;
- uintptr_t *tb_next;
- uint16_t *tb_next_offset;
- uint16_t *tb_jmp_offset; /* != NULL if USE_DIRECT_JUMP */
+ uint16_t *tb_jmp_reset_offset; /* tb->jmp_reset_offset */
+ uint16_t *tb_jmp_insn_offset; /* tb->jmp_insn_offset if USE_DIRECT_JUMP */
+ uintptr_t *tb_jmp_target_addr; /* tb->jmp_target_addr if !USE_DIRECT_JUMP */
/* liveness analysis */
uint16_t *op_dead_args; /* for each operation, each bit tells if the
@@ -925,7 +925,7 @@ static inline unsigned get_mmuidx(TCGMemOpIdx oi)
/**
* tcg_qemu_tb_exec:
- * @env: CPUArchState * for the CPU
+ * @env: pointer to CPUArchState for the CPU
* @tb_ptr: address of generated code for the TB to execute
*
* Start executing code from a given translation block.
@@ -936,30 +936,31 @@ static inline unsigned get_mmuidx(TCGMemOpIdx oi)
* which has not yet been directly linked, or an asynchronous
* event such as an interrupt needs handling.
*
- * The return value is a pointer to the next TB to execute
- * (if known; otherwise zero). This pointer is assumed to be
- * 4-aligned, and the bottom two bits are used to return further
- * information:
+ * Return: The return value is the value passed to the corresponding
+ * tcg_gen_exit_tb() at translation time of the last TB attempted to execute.
+ * The value is either zero or a 4-byte aligned pointer to that TB combined
+ * with additional information in its two least significant bits. The
+ * additional information is encoded as follows:
* 0, 1: the link between this TB and the next is via the specified
* TB index (0 or 1). That is, we left the TB via (the equivalent
* of) "goto_tb <index>". The main loop uses this to determine
* how to link the TB just executed to the next.
* 2: we are using instruction counting code generation, and we
* did not start executing this TB because the instruction counter
- * would hit zero midway through it. In this case the next-TB pointer
+ * would hit zero midway through it. In this case the pointer
* returned is the TB we were about to execute, and the caller must
* arrange to execute the remaining count of instructions.
* 3: we stopped because the CPU's exit_request flag was set
* (usually meaning that there is an interrupt that needs to be
- * handled). The next-TB pointer returned is the TB we were
- * about to execute when we noticed the pending exit request.
+ * handled). The pointer returned is the TB we were about to execute
+ * when we noticed the pending exit request.
*
* If the bottom two bits indicate an exit-via-index then the CPU
* state is correctly synchronised and ready for execution of the next
* TB (and in particular the guest PC is the address to execute next).
* Otherwise, we gave up on execution of this TB before it started, and
* the caller must fix up the CPU state by calling the CPU's
- * synchronize_from_tb() method with the next-TB pointer we return (falling
+ * synchronize_from_tb() method with the TB pointer we return (falling
* back to calling the CPU's set_pc method with tb->pb if no
* synchronize_from_tb() method exists).
*
diff --git a/tcg/tci/tcg-target.inc.c b/tcg/tci/tcg-target.inc.c
index e2fc52a167..fa74d5278e 100644
--- a/tcg/tci/tcg-target.inc.c
+++ b/tcg/tci/tcg-target.inc.c
@@ -553,17 +553,19 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
tcg_out64(s, args[0]);
break;
case INDEX_op_goto_tb:
- if (s->tb_jmp_offset) {
+ if (s->tb_jmp_insn_offset) {
/* Direct jump method. */
- tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_jmp_offset));
- s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
+ tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_jmp_insn_offset));
+ /* Align for atomic patching and thread safety */
+ s->code_ptr = QEMU_ALIGN_PTR_UP(s->code_ptr, 4);
+ s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
tcg_out32(s, 0);
} else {
/* Indirect jump method. */
TODO();
}
- tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_next_offset));
- s->tb_next_offset[args[0]] = tcg_current_code_size(s);
+ tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_jmp_reset_offset));
+ s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
break;
case INDEX_op_br:
tci_out_label(s, arg_label(args[0]));
diff --git a/tci.c b/tci.c
index 82705fe772..0fdc4e2c12 100644
--- a/tci.c
+++ b/tci.c
@@ -467,7 +467,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
{
long tcg_temps[CPU_TEMP_BUF_NLONGS];
uintptr_t sp_value = (uintptr_t)(tcg_temps + CPU_TEMP_BUF_NLONGS);
- uintptr_t next_tb = 0;
+ uintptr_t ret = 0;
tci_reg[TCG_AREG0] = (tcg_target_ulong)env;
tci_reg[TCG_REG_CALL_STACK] = sp_value;
@@ -1085,11 +1085,14 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
/* QEMU specific operations. */
case INDEX_op_exit_tb:
- next_tb = *(uint64_t *)tb_ptr;
+ ret = *(uint64_t *)tb_ptr;
goto exit;
break;
case INDEX_op_goto_tb:
- t0 = tci_read_i32(&tb_ptr);
+ /* Jump address is aligned */
+ tb_ptr = QEMU_ALIGN_PTR_UP(tb_ptr, 4);
+ t0 = atomic_read((int32_t *)tb_ptr);
+ tb_ptr += sizeof(int32_t);
tci_assert(tb_ptr == old_code_ptr + op_size);
tb_ptr += (int32_t)t0;
continue;
@@ -1240,5 +1243,5 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
tci_assert(tb_ptr == old_code_ptr + op_size);
}
exit:
- return next_tb;
+ return ret;
}
diff --git a/trace-events b/trace-events
index b588091b6c..4fce005afd 100644
--- a/trace-events
+++ b/trace-events
@@ -1614,7 +1614,7 @@ kvm_failed_spr_get(int str, const char *msg) "Warning: Unable to retrieve SPR %d
# cpu-exec.c
disable exec_tb(void *tb, uintptr_t pc) "tb:%p pc=0x%"PRIxPTR
disable exec_tb_nocache(void *tb, uintptr_t pc) "tb:%p pc=0x%"PRIxPTR
-disable exec_tb_exit(void *next_tb, unsigned int flags) "tb:%p flags=%x"
+disable exec_tb_exit(void *last_tb, unsigned int flags) "tb:%p flags=%x"
# translate-all.c
translate_block(void *tb, uintptr_t pc, uint8_t *tb_code) "tb:%p, pc:0x%"PRIxPTR", tb_code:%p"
diff --git a/translate-all.c b/translate-all.c
index 8329ea60ee..b54f472531 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -72,11 +72,12 @@
typedef struct PageDesc {
/* list of TBs intersecting this ram page */
TranslationBlock *first_tb;
+#ifdef CONFIG_SOFTMMU
/* in order to optimize self modifying code, we count the number
of lookups we do to a given page to use a bitmap */
unsigned int code_write_count;
unsigned long *code_bitmap;
-#if defined(CONFIG_USER_ONLY)
+#else
unsigned long flags;
#endif
} PageDesc;
@@ -153,8 +154,6 @@ void tb_lock_reset(void)
#endif
}
-static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
- tb_page_addr_t phys_page2);
static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
void cpu_gen_init(void)
@@ -306,7 +305,6 @@ bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
cpu_restore_state_from_tb(cpu, tb, retaddr);
if (tb->cflags & CF_NOCACHE) {
/* one-shot translation, invalidate it immediately */
- cpu->current_tb = NULL;
tb_phys_invalidate(tb, -1);
tb_free(tb);
}
@@ -464,6 +462,8 @@ static inline PageDesc *page_find(tb_page_addr_t index)
# define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
#elif defined(__powerpc64__)
# define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
+#elif defined(__powerpc__)
+# define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
#elif defined(__aarch64__)
# define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
#elif defined(__arm__)
@@ -505,7 +505,6 @@ static inline size_t size_code_gen_buffer(size_t tb_size)
if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
tb_size = MAX_CODE_GEN_BUFFER_SIZE;
}
- tcg_ctx.code_gen_buffer_size = tb_size;
return tb_size;
}
@@ -514,7 +513,7 @@ static inline size_t size_code_gen_buffer(size_t tb_size)
that the buffer not cross a 256MB boundary. */
static inline bool cross_256mb(void *addr, size_t size)
{
- return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & 0xf0000000;
+ return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
}
/* We weren't able to allocate a buffer without crossing that boundary,
@@ -522,7 +521,7 @@ static inline bool cross_256mb(void *addr, size_t size)
Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
static inline void *split_cross_256mb(void *buf1, size_t size1)
{
- void *buf2 = (void *)(((uintptr_t)buf1 + size1) & 0xf0000000);
+ void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
size_t size2 = buf1 + size1 - buf2;
size1 = buf2 - buf1;
@@ -683,11 +682,11 @@ static inline void *alloc_code_gen_buffer(void)
case 1:
if (!cross_256mb(buf2, size)) {
/* Success! Use the new buffer. */
- munmap(buf, size);
+ munmap(buf, size + qemu_real_host_page_size);
break;
}
/* Failure. Work with what we had. */
- munmap(buf2, size);
+ munmap(buf2, size + qemu_real_host_page_size);
/* fallthru */
default:
/* Split the original buffer. Free the smaller half. */
@@ -784,9 +783,11 @@ void tb_free(TranslationBlock *tb)
static inline void invalidate_page_bitmap(PageDesc *p)
{
+#ifdef CONFIG_SOFTMMU
g_free(p->code_bitmap);
p->code_bitmap = NULL;
p->code_write_count = 0;
+#endif
}
/* Set to NULL all the 'first_tb' fields in all PageDescs. */
@@ -841,6 +842,7 @@ void tb_flush(CPUState *cpu)
CPU_FOREACH(cpu) {
memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
+ cpu->tb_flushed = true;
}
memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
@@ -925,32 +927,33 @@ static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
}
}
-static inline void tb_jmp_remove(TranslationBlock *tb, int n)
+/* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
+static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n)
{
- TranslationBlock *tb1, **ptb;
+ TranslationBlock *tb1;
+ uintptr_t *ptb, ntb;
unsigned int n1;
- ptb = &tb->jmp_next[n];
- tb1 = *ptb;
- if (tb1) {
+ ptb = &tb->jmp_list_next[n];
+ if (*ptb) {
/* find tb(n) in circular list */
for (;;) {
- tb1 = *ptb;
- n1 = (uintptr_t)tb1 & 3;
- tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+ ntb = *ptb;
+ n1 = ntb & 3;
+ tb1 = (TranslationBlock *)(ntb & ~3);
if (n1 == n && tb1 == tb) {
break;
}
if (n1 == 2) {
- ptb = &tb1->jmp_first;
+ ptb = &tb1->jmp_list_first;
} else {
- ptb = &tb1->jmp_next[n1];
+ ptb = &tb1->jmp_list_next[n1];
}
}
/* now we can suppress tb(n) from the list */
- *ptb = tb->jmp_next[n];
+ *ptb = tb->jmp_list_next[n];
- tb->jmp_next[n] = NULL;
+ tb->jmp_list_next[n] = (uintptr_t)NULL;
}
}
@@ -958,7 +961,29 @@ static inline void tb_jmp_remove(TranslationBlock *tb, int n)
another TB */
static inline void tb_reset_jump(TranslationBlock *tb, int n)
{
- tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
+ uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]);
+ tb_set_jmp_target(tb, n, addr);
+}
+
+/* remove any jumps to the TB */
+static inline void tb_jmp_unlink(TranslationBlock *tb)
+{
+ TranslationBlock *tb1;
+ uintptr_t *ptb, ntb;
+ unsigned int n1;
+
+ ptb = &tb->jmp_list_first;
+ for (;;) {
+ ntb = *ptb;
+ n1 = ntb & 3;
+ tb1 = (TranslationBlock *)(ntb & ~3);
+ if (n1 == 2) {
+ break;
+ }
+ tb_reset_jump(tb1, n1);
+ *ptb = tb1->jmp_list_next[n1];
+ tb1->jmp_list_next[n1] = (uintptr_t)NULL;
+ }
}
/* invalidate one TB */
@@ -966,9 +991,8 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
{
CPUState *cpu;
PageDesc *p;
- unsigned int h, n1;
+ unsigned int h;
tb_page_addr_t phys_pc;
- TranslationBlock *tb1, *tb2;
/* remove the TB from the hash list */
phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
@@ -987,8 +1011,6 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
invalidate_page_bitmap(p);
}
- tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
-
/* remove the TB from the hash list */
h = tb_jmp_cache_hash_func(tb->pc);
CPU_FOREACH(cpu) {
@@ -998,27 +1020,16 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
}
/* suppress this TB from the two jump lists */
- tb_jmp_remove(tb, 0);
- tb_jmp_remove(tb, 1);
+ tb_remove_from_jmp_list(tb, 0);
+ tb_remove_from_jmp_list(tb, 1);
/* suppress any remaining jumps to this TB */
- tb1 = tb->jmp_first;
- for (;;) {
- n1 = (uintptr_t)tb1 & 3;
- if (n1 == 2) {
- break;
- }
- tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
- tb2 = tb1->jmp_next[n1];
- tb_reset_jump(tb1, n1);
- tb1->jmp_next[n1] = NULL;
- tb1 = tb2;
- }
- tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
+ tb_jmp_unlink(tb);
tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
}
+#ifdef CONFIG_SOFTMMU
static void build_page_bitmap(PageDesc *p)
{
int n, tb_start, tb_end;
@@ -1047,11 +1058,100 @@ static void build_page_bitmap(PageDesc *p)
tb = tb->page_next[n];
}
}
+#endif
+
+/* add the tb in the target page and protect it if necessary
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
+static inline void tb_alloc_page(TranslationBlock *tb,
+ unsigned int n, tb_page_addr_t page_addr)
+{
+ PageDesc *p;
+#ifndef CONFIG_USER_ONLY
+ bool page_already_protected;
+#endif
+
+ tb->page_addr[n] = page_addr;
+ p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
+ tb->page_next[n] = p->first_tb;
+#ifndef CONFIG_USER_ONLY
+ page_already_protected = p->first_tb != NULL;
+#endif
+ p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
+ invalidate_page_bitmap(p);
+
+#if defined(CONFIG_USER_ONLY)
+ if (p->flags & PAGE_WRITE) {
+ target_ulong addr;
+ PageDesc *p2;
+ int prot;
+
+ /* force the host page as non writable (writes will have a
+ page fault + mprotect overhead) */
+ page_addr &= qemu_host_page_mask;
+ prot = 0;
+ for (addr = page_addr; addr < page_addr + qemu_host_page_size;
+ addr += TARGET_PAGE_SIZE) {
+
+ p2 = page_find(addr >> TARGET_PAGE_BITS);
+ if (!p2) {
+ continue;
+ }
+ prot |= p2->flags;
+ p2->flags &= ~PAGE_WRITE;
+ }
+ mprotect(g2h(page_addr), qemu_host_page_size,
+ (prot & PAGE_BITS) & ~PAGE_WRITE);
+#ifdef DEBUG_TB_INVALIDATE
+ printf("protecting code page: 0x" TARGET_FMT_lx "\n",
+ page_addr);
+#endif
+ }
+#else
+ /* if some code is already present, then the pages are already
+ protected. So we handle the case where only the first TB is
+ allocated in a physical page */
+ if (!page_already_protected) {
+ tlb_protect_code(page_addr);
+ }
+#endif
+}
+
+/* add a new TB and link it to the physical page tables. phys_page2 is
+ * (-1) to indicate that only one page contains the TB.
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
+static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+ tb_page_addr_t phys_page2)
+{
+ unsigned int h;
+ TranslationBlock **ptb;
+
+ /* add in the physical hash table */
+ h = tb_phys_hash_func(phys_pc);
+ ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
+ tb->phys_hash_next = *ptb;
+ *ptb = tb;
+
+ /* add in the page list */
+ tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
+ if (phys_page2 != -1) {
+ tb_alloc_page(tb, 1, phys_page2);
+ } else {
+ tb->page_addr[1] = -1;
+ }
+
+#ifdef DEBUG_TB_CHECK
+ tb_page_check();
+#endif
+}
/* Called with mmap_lock held for user mode emulation. */
TranslationBlock *tb_gen_code(CPUState *cpu,
target_ulong pc, target_ulong cs_base,
- int flags, int cflags)
+ uint32_t flags, int cflags)
{
CPUArchState *env = cpu->env_ptr;
TranslationBlock *tb;
@@ -1076,8 +1176,6 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
/* cannot fail at this point */
tb = tb_alloc(pc);
assert(tb != NULL);
- /* Don't forget to invalidate previous TB info. */
- tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
}
gen_code_buf = tcg_ctx.code_gen_ptr;
@@ -1099,15 +1197,15 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
trace_translate_block(tb, tb->pc, tb->tc_ptr);
/* generate machine code */
- tb->tb_next_offset[0] = 0xffff;
- tb->tb_next_offset[1] = 0xffff;
- tcg_ctx.tb_next_offset = tb->tb_next_offset;
+ tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
+ tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
+ tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset;
#ifdef USE_DIRECT_JUMP
- tcg_ctx.tb_jmp_offset = tb->tb_jmp_offset;
- tcg_ctx.tb_next = NULL;
+ tcg_ctx.tb_jmp_insn_offset = tb->jmp_insn_offset;
+ tcg_ctx.tb_jmp_target_addr = NULL;
#else
- tcg_ctx.tb_jmp_offset = NULL;
- tcg_ctx.tb_next = tb->tb_next;
+ tcg_ctx.tb_jmp_insn_offset = NULL;
+ tcg_ctx.tb_jmp_target_addr = tb->jmp_target_addr;
#endif
#ifdef CONFIG_PROFILER
@@ -1151,12 +1249,31 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
CODE_GEN_ALIGN);
+ /* init jump list */
+ assert(((uintptr_t)tb & 3) == 0);
+ tb->jmp_list_first = (uintptr_t)tb | 2;
+ tb->jmp_list_next[0] = (uintptr_t)NULL;
+ tb->jmp_list_next[1] = (uintptr_t)NULL;
+
+ /* init original jump addresses wich has been set during tcg_gen_code() */
+ if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
+ tb_reset_jump(tb, 0);
+ }
+ if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
+ tb_reset_jump(tb, 1);
+ }
+
/* check next page if needed */
virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
phys_page2 = -1;
if ((pc & TARGET_PAGE_MASK) != virt_page2) {
phys_page2 = get_page_addr_code(env, virt_page2);
}
+ /* As long as consistency of the TB stuff is provided by tb_lock in user
+ * mode and is implicit in single-threaded softmmu emulation, no explicit
+ * memory barrier is required before tb_link_page() makes the TB visible
+ * through the physical hash table and physical page list.
+ */
tb_link_page(tb, phys_pc, phys_page2);
return tb;
}
@@ -1191,9 +1308,9 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
int is_cpu_write_access)
{
- TranslationBlock *tb, *tb_next, *saved_tb;
- CPUState *cpu = current_cpu;
+ TranslationBlock *tb, *tb_next;
#if defined(TARGET_HAS_PRECISE_SMC)
+ CPUState *cpu = current_cpu;
CPUArchState *env = NULL;
#endif
tb_page_addr_t tb_start, tb_end;
@@ -1205,7 +1322,7 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
int current_tb_modified = 0;
target_ulong current_pc = 0;
target_ulong current_cs_base = 0;
- int current_flags = 0;
+ uint32_t current_flags = 0;
#endif /* TARGET_HAS_PRECISE_SMC */
p = page_find(start >> TARGET_PAGE_BITS);
@@ -1260,20 +1377,7 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
&current_flags);
}
#endif /* TARGET_HAS_PRECISE_SMC */
- /* we need to do that to handle the case where a signal
- occurs while doing tb_phys_invalidate() */
- saved_tb = NULL;
- if (cpu != NULL) {
- saved_tb = cpu->current_tb;
- cpu->current_tb = NULL;
- }
tb_phys_invalidate(tb, -1);
- if (cpu != NULL) {
- cpu->current_tb = saved_tb;
- if (cpu->interrupt_request && cpu->current_tb) {
- cpu_interrupt(cpu, cpu->interrupt_request);
- }
- }
}
tb = tb_next;
}
@@ -1289,13 +1393,13 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
/* we generate a block containing just the instruction
modifying the memory. It will ensure that it cannot modify
itself */
- cpu->current_tb = NULL;
tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
cpu_resume_from_signal(cpu, NULL);
}
#endif
}
+#ifdef CONFIG_SOFTMMU
/* len must be <= 8 and start must be a multiple of len */
void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
{
@@ -1333,8 +1437,7 @@ void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
tb_invalidate_phys_page_range(start, start + len, 1);
}
}
-
-#if !defined(CONFIG_SOFTMMU)
+#else
/* Called with mmap_lock held. */
static void tb_invalidate_phys_page(tb_page_addr_t addr,
uintptr_t pc, void *puc,
@@ -1350,7 +1453,7 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr,
int current_tb_modified = 0;
target_ulong current_pc = 0;
target_ulong current_cs_base = 0;
- int current_flags = 0;
+ uint32_t current_flags = 0;
#endif
addr &= TARGET_PAGE_MASK;
@@ -1394,7 +1497,6 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr,
/* we generate a block containing just the instruction
modifying the memory. It will ensure that it cannot modify
itself */
- cpu->current_tb = NULL;
tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
if (locked) {
mmap_unlock();
@@ -1405,106 +1507,6 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr,
}
#endif
-/* add the tb in the target page and protect it if necessary
- *
- * Called with mmap_lock held for user-mode emulation.
- */
-static inline void tb_alloc_page(TranslationBlock *tb,
- unsigned int n, tb_page_addr_t page_addr)
-{
- PageDesc *p;
-#ifndef CONFIG_USER_ONLY
- bool page_already_protected;
-#endif
-
- tb->page_addr[n] = page_addr;
- p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
- tb->page_next[n] = p->first_tb;
-#ifndef CONFIG_USER_ONLY
- page_already_protected = p->first_tb != NULL;
-#endif
- p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
- invalidate_page_bitmap(p);
-
-#if defined(CONFIG_USER_ONLY)
- if (p->flags & PAGE_WRITE) {
- target_ulong addr;
- PageDesc *p2;
- int prot;
-
- /* force the host page as non writable (writes will have a
- page fault + mprotect overhead) */
- page_addr &= qemu_host_page_mask;
- prot = 0;
- for (addr = page_addr; addr < page_addr + qemu_host_page_size;
- addr += TARGET_PAGE_SIZE) {
-
- p2 = page_find(addr >> TARGET_PAGE_BITS);
- if (!p2) {
- continue;
- }
- prot |= p2->flags;
- p2->flags &= ~PAGE_WRITE;
- }
- mprotect(g2h(page_addr), qemu_host_page_size,
- (prot & PAGE_BITS) & ~PAGE_WRITE);
-#ifdef DEBUG_TB_INVALIDATE
- printf("protecting code page: 0x" TARGET_FMT_lx "\n",
- page_addr);
-#endif
- }
-#else
- /* if some code is already present, then the pages are already
- protected. So we handle the case where only the first TB is
- allocated in a physical page */
- if (!page_already_protected) {
- tlb_protect_code(page_addr);
- }
-#endif
-}
-
-/* add a new TB and link it to the physical page tables. phys_page2 is
- * (-1) to indicate that only one page contains the TB.
- *
- * Called with mmap_lock held for user-mode emulation.
- */
-static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
- tb_page_addr_t phys_page2)
-{
- unsigned int h;
- TranslationBlock **ptb;
-
- /* add in the physical hash table */
- h = tb_phys_hash_func(phys_pc);
- ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
- tb->phys_hash_next = *ptb;
- *ptb = tb;
-
- /* add in the page list */
- tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
- if (phys_page2 != -1) {
- tb_alloc_page(tb, 1, phys_page2);
- } else {
- tb->page_addr[1] = -1;
- }
-
- tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
- tb->jmp_next[0] = NULL;
- tb->jmp_next[1] = NULL;
-
- /* init original jump addresses */
- if (tb->tb_next_offset[0] != 0xffff) {
- tb_reset_jump(tb, 0);
- }
- if (tb->tb_next_offset[1] != 0xffff) {
- tb_reset_jump(tb, 1);
- }
-
-#ifdef DEBUG_TB_CHECK
- tb_page_check();
-#endif
-}
-
/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
tb[1].tc_ptr. Return NULL if not found */
static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
@@ -1574,7 +1576,7 @@ void tb_check_watchpoint(CPUState *cpu)
CPUArchState *env = cpu->env_ptr;
target_ulong pc, cs_base;
tb_page_addr_t addr;
- int flags;
+ uint32_t flags;
cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
addr = get_page_addr_code(env, pc);
@@ -1593,7 +1595,7 @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
TranslationBlock *tb;
uint32_t n, cflags;
target_ulong pc, cs_base;
- uint64_t flags;
+ uint32_t flags;
tb = tb_find_pc(retaddr);
if (!tb) {
@@ -1689,9 +1691,9 @@ void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
if (tb->page_addr[1] != -1) {
cross_page++;
}
- if (tb->tb_next_offset[0] != 0xffff) {
+ if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
direct_jmp_count++;
- if (tb->tb_next_offset[1] != 0xffff) {
+ if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
direct_jmp2_count++;
}
}