aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPeter Maydell <peter.maydell@linaro.org>2015-07-06 14:03:44 +0100
committerPeter Maydell <peter.maydell@linaro.org>2015-07-06 14:03:44 +0100
commit7edd8e4660beb301d527257f8e04ebec0f841cb0 (patch)
tree49ed5104cd81447d68c9131dbface7bf8781ac61
parent3fa18bc9a55e067ba3012ab1d394f5d5a7e51419 (diff)
parentb242e0e0e2969c044a318e56f7988bbd84de1f63 (diff)
downloadqemu-arm-7edd8e4660beb301d527257f8e04ebec0f841cb0.tar.gz
Merge remote-tracking branch 'remotes/bonzini/tags/for-upstream' into staging
* more of Peter Crosthwaite's multiarch preparation patches * unlocked MMIO support in KVM * support for compilation with ICC # gpg: Signature made Mon Jul 6 13:59:20 2015 BST using RSA key ID 78C7AE83 # gpg: Good signature from "Paolo Bonzini <bonzini@gnu.org>" # gpg: aka "Paolo Bonzini <pbonzini@redhat.com>" # gpg: WARNING: This key is not certified with sufficiently trusted signatures! # gpg: It is not certain that the signature belongs to the owner. # Primary key fingerprint: 46F5 9FBD 57D6 12E7 BFD4 E2F7 7E15 100C CD36 69B1 # Subkey fingerprint: F133 3857 4B66 2389 866C 7682 BFFB D25F 78C7 AE83 * remotes/bonzini/tags/for-upstream: exec: skip MMIO regions correctly in cpu_physical_memory_write_rom_internal Stop including qemu-common.h in memory.h kvm: Switch to unlocked MMIO acpi: mark PMTIMER as unlocked kvm: Switch to unlocked PIO kvm: First step to push iothread lock out of inner run loop memory: let address_space_rw/ld*/st* run outside the BQL exec: pull qemu_flush_coalesced_mmio_buffer() into address_space_rw/ld*/st* memory: Add global-locking property to memory regions main-loop: introduce qemu_mutex_iothread_locked main-loop: use qemu_mutex_lock_iothread consistently Fix irq route entries exceeding KVM_MAX_IRQ_ROUTES cpu-defs: Move out TB_JMP defines include/exec: Move tb hash functions out include/exec: Move standard exceptions to cpu-all.h cpu-defs: Move CPU_TEMP_BUF_NLONGS to tcg memory_mapping: Rework cpu related includes cutils: allow compilation with icc qemu-common: add VEC_OR macro Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
-rw-r--r--cpu-exec.c1
-rw-r--r--cpus.c19
-rw-r--r--exec.c83
-rw-r--r--hw/acpi/core.c1
-rw-r--r--include/exec/cpu-all.h6
-rw-r--r--include/exec/cpu-common.h2
-rw-r--r--include/exec/cpu-defs.h15
-rw-r--r--include/exec/exec-all.h20
-rw-r--r--include/exec/memory.h27
-rw-r--r--include/exec/tb-hash.h51
-rw-r--r--include/hw/arm/arm.h1
-rw-r--r--include/qemu-common.h7
-rw-r--r--include/qemu/fprintf-fn.h17
-rw-r--r--include/qemu/main-loop.h10
-rw-r--r--kvm-all.c25
-rw-r--r--memory.c23
-rw-r--r--memory_mapping.c2
-rw-r--r--stubs/iothread-lock.c5
-rw-r--r--target-i386/kvm.c24
-rw-r--r--target-mips/kvm.c4
-rw-r--r--target-ppc/kvm.c7
-rw-r--r--target-s390x/kvm.c3
-rw-r--r--target-s390x/mmu_helper.c2
-rw-r--r--tcg/tcg.h2
-rw-r--r--translate-all.c1
-rw-r--r--util/cutils.c14
26 files changed, 299 insertions, 73 deletions
diff --git a/cpu-exec.c b/cpu-exec.c
index 2ffeb6e40d..b2724c18c1 100644
--- a/cpu-exec.c
+++ b/cpu-exec.c
@@ -27,6 +27,7 @@
#include "exec/address-spaces.h"
#include "exec/memory-internal.h"
#include "qemu/rcu.h"
+#include "exec/tb-hash.h"
/* -icount align implementation. */
diff --git a/cpus.c b/cpus.c
index 4f0e54d53c..f547aebeaf 100644
--- a/cpus.c
+++ b/cpus.c
@@ -954,7 +954,7 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
CPUState *cpu = arg;
int r;
- qemu_mutex_lock(&qemu_global_mutex);
+ qemu_mutex_lock_iothread();
qemu_thread_get_self(cpu->thread);
cpu->thread_id = qemu_get_thread_id();
cpu->can_do_io = 1;
@@ -1034,10 +1034,10 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
{
CPUState *cpu = arg;
+ qemu_mutex_lock_iothread();
qemu_tcg_init_cpu_signals();
qemu_thread_get_self(cpu->thread);
- qemu_mutex_lock(&qemu_global_mutex);
CPU_FOREACH(cpu) {
cpu->thread_id = qemu_get_thread_id();
cpu->created = true;
@@ -1146,10 +1146,21 @@ bool qemu_in_vcpu_thread(void)
return current_cpu && qemu_cpu_is_self(current_cpu);
}
+static __thread bool iothread_locked = false;
+
+bool qemu_mutex_iothread_locked(void)
+{
+ return iothread_locked;
+}
+
void qemu_mutex_lock_iothread(void)
{
atomic_inc(&iothread_requesting_mutex);
- if (!tcg_enabled() || !first_cpu || !first_cpu->thread) {
+ /* In the simple case there is no need to bump the VCPU thread out of
+ * TCG code execution.
+ */
+ if (!tcg_enabled() || qemu_in_vcpu_thread() ||
+ !first_cpu || !first_cpu->thread) {
qemu_mutex_lock(&qemu_global_mutex);
atomic_dec(&iothread_requesting_mutex);
} else {
@@ -1160,10 +1171,12 @@ void qemu_mutex_lock_iothread(void)
atomic_dec(&iothread_requesting_mutex);
qemu_cond_broadcast(&qemu_io_proceeded_cond);
}
+ iothread_locked = true;
}
void qemu_mutex_unlock_iothread(void)
{
+ iothread_locked = false;
qemu_mutex_unlock(&qemu_global_mutex);
}
diff --git a/exec.c b/exec.c
index f7883d2246..251dc79e10 100644
--- a/exec.c
+++ b/exec.c
@@ -48,6 +48,7 @@
#endif
#include "exec/cpu-all.h"
#include "qemu/rcu_queue.h"
+#include "qemu/main-loop.h"
#include "exec/cputlb.h"
#include "translate-all.h"
@@ -352,6 +353,18 @@ address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *x
*xlat = addr + section->offset_within_region;
mr = section->mr;
+
+ /* MMIO registers can be expected to perform full-width accesses based only
+ * on their address, without considering adjacent registers that could
+ * decode to completely different MemoryRegions. When such registers
+ * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
+ * regions overlap wildly. For this reason we cannot clamp the accesses
+ * here.
+ *
+ * If the length is small (as is the case for address_space_ldl/stl),
+ * everything works fine. If the incoming length is large, however,
+ * the caller really has to do the clamping through memory_access_size.
+ */
if (memory_region_is_ram(mr)) {
diff = int128_sub(section->size, int128_make64(addr));
*plen = int128_get64(int128_min(diff, int128_make64(*plen)));
@@ -2316,6 +2329,29 @@ static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
return l;
}
+static bool prepare_mmio_access(MemoryRegion *mr)
+{
+ bool unlocked = !qemu_mutex_iothread_locked();
+ bool release_lock = false;
+
+ if (unlocked && mr->global_locking) {
+ qemu_mutex_lock_iothread();
+ unlocked = false;
+ release_lock = true;
+ }
+ if (mr->flush_coalesced_mmio) {
+ if (unlocked) {
+ qemu_mutex_lock_iothread();
+ }
+ qemu_flush_coalesced_mmio_buffer();
+ if (unlocked) {
+ qemu_mutex_unlock_iothread();
+ }
+ }
+
+ return release_lock;
+}
+
MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
uint8_t *buf, int len, bool is_write)
{
@@ -2325,6 +2361,7 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
hwaddr addr1;
MemoryRegion *mr;
MemTxResult result = MEMTX_OK;
+ bool release_lock = false;
rcu_read_lock();
while (len > 0) {
@@ -2333,6 +2370,7 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
if (is_write) {
if (!memory_access_is_direct(mr, is_write)) {
+ release_lock |= prepare_mmio_access(mr);
l = memory_access_size(mr, l, addr1);
/* XXX: could force current_cpu to NULL to avoid
potential bugs */
@@ -2374,6 +2412,7 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
} else {
if (!memory_access_is_direct(mr, is_write)) {
/* I/O case */
+ release_lock |= prepare_mmio_access(mr);
l = memory_access_size(mr, l, addr1);
switch (l) {
case 8:
@@ -2409,6 +2448,12 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
memcpy(buf, ptr, l);
}
}
+
+ if (release_lock) {
+ qemu_mutex_unlock_iothread();
+ release_lock = false;
+ }
+
len -= l;
buf += l;
addr += l;
@@ -2458,7 +2503,7 @@ static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
if (!(memory_region_is_ram(mr) ||
memory_region_is_romd(mr))) {
- /* do nothing */
+ l = memory_access_size(mr, l, addr1);
} else {
addr1 += memory_region_get_ram_addr(mr);
/* ROM/RAM case */
@@ -2735,10 +2780,13 @@ static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
hwaddr l = 4;
hwaddr addr1;
MemTxResult r;
+ bool release_lock = false;
rcu_read_lock();
mr = address_space_translate(as, addr, &addr1, &l, false);
if (l < 4 || !memory_access_is_direct(mr, false)) {
+ release_lock |= prepare_mmio_access(mr);
+
/* I/O case */
r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
#if defined(TARGET_WORDS_BIGENDIAN)
@@ -2771,6 +2819,9 @@ static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
if (result) {
*result = r;
}
+ if (release_lock) {
+ qemu_mutex_unlock_iothread();
+ }
rcu_read_unlock();
return val;
}
@@ -2823,11 +2874,14 @@ static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
hwaddr l = 8;
hwaddr addr1;
MemTxResult r;
+ bool release_lock = false;
rcu_read_lock();
mr = address_space_translate(as, addr, &addr1, &l,
false);
if (l < 8 || !memory_access_is_direct(mr, false)) {
+ release_lock |= prepare_mmio_access(mr);
+
/* I/O case */
r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
#if defined(TARGET_WORDS_BIGENDIAN)
@@ -2860,6 +2914,9 @@ static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
if (result) {
*result = r;
}
+ if (release_lock) {
+ qemu_mutex_unlock_iothread();
+ }
rcu_read_unlock();
return val;
}
@@ -2932,11 +2989,14 @@ static inline uint32_t address_space_lduw_internal(AddressSpace *as,
hwaddr l = 2;
hwaddr addr1;
MemTxResult r;
+ bool release_lock = false;
rcu_read_lock();
mr = address_space_translate(as, addr, &addr1, &l,
false);
if (l < 2 || !memory_access_is_direct(mr, false)) {
+ release_lock |= prepare_mmio_access(mr);
+
/* I/O case */
r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
#if defined(TARGET_WORDS_BIGENDIAN)
@@ -2969,6 +3029,9 @@ static inline uint32_t address_space_lduw_internal(AddressSpace *as,
if (result) {
*result = r;
}
+ if (release_lock) {
+ qemu_mutex_unlock_iothread();
+ }
rcu_read_unlock();
return val;
}
@@ -3021,11 +3084,14 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
hwaddr addr1;
MemTxResult r;
uint8_t dirty_log_mask;
+ bool release_lock = false;
rcu_read_lock();
mr = address_space_translate(as, addr, &addr1, &l,
true);
if (l < 4 || !memory_access_is_direct(mr, true)) {
+ release_lock |= prepare_mmio_access(mr);
+
r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
} else {
addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
@@ -3040,6 +3106,9 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
if (result) {
*result = r;
}
+ if (release_lock) {
+ qemu_mutex_unlock_iothread();
+ }
rcu_read_unlock();
}
@@ -3060,11 +3129,14 @@ static inline void address_space_stl_internal(AddressSpace *as,
hwaddr l = 4;
hwaddr addr1;
MemTxResult r;
+ bool release_lock = false;
rcu_read_lock();
mr = address_space_translate(as, addr, &addr1, &l,
true);
if (l < 4 || !memory_access_is_direct(mr, true)) {
+ release_lock |= prepare_mmio_access(mr);
+
#if defined(TARGET_WORDS_BIGENDIAN)
if (endian == DEVICE_LITTLE_ENDIAN) {
val = bswap32(val);
@@ -3096,6 +3168,9 @@ static inline void address_space_stl_internal(AddressSpace *as,
if (result) {
*result = r;
}
+ if (release_lock) {
+ qemu_mutex_unlock_iothread();
+ }
rcu_read_unlock();
}
@@ -3165,10 +3240,13 @@ static inline void address_space_stw_internal(AddressSpace *as,
hwaddr l = 2;
hwaddr addr1;
MemTxResult r;
+ bool release_lock = false;
rcu_read_lock();
mr = address_space_translate(as, addr, &addr1, &l, true);
if (l < 2 || !memory_access_is_direct(mr, true)) {
+ release_lock |= prepare_mmio_access(mr);
+
#if defined(TARGET_WORDS_BIGENDIAN)
if (endian == DEVICE_LITTLE_ENDIAN) {
val = bswap16(val);
@@ -3200,6 +3278,9 @@ static inline void address_space_stw_internal(AddressSpace *as,
if (result) {
*result = r;
}
+ if (release_lock) {
+ qemu_mutex_unlock_iothread();
+ }
rcu_read_unlock();
}
diff --git a/hw/acpi/core.c b/hw/acpi/core.c
index 0f201d8c6d..fe6215af4a 100644
--- a/hw/acpi/core.c
+++ b/hw/acpi/core.c
@@ -528,6 +528,7 @@ void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci,
ar->tmr.timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, acpi_pm_tmr_timer, ar);
memory_region_init_io(&ar->tmr.io, memory_region_owner(parent),
&acpi_pm_tmr_ops, ar, "acpi-tmr", 4);
+ memory_region_clear_global_locking(&ar->tmr.io);
memory_region_add_subregion(parent, 8, &ar->tmr.io);
}
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index ac06c6721c..8999634981 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -26,6 +26,12 @@
#include "qom/cpu.h"
#include "qemu/rcu.h"
+#define EXCP_INTERRUPT 0x10000 /* async interruption */
+#define EXCP_HLT 0x10001 /* hlt instruction reached */
+#define EXCP_DEBUG 0x10002 /* cpu stopped after a breakpoint or singlestep */
+#define EXCP_HALTED 0x10003 /* cpu is halted (waiting for external event) */
+#define EXCP_YIELD 0x10004 /* cpu wants to yield timeslice to another */
+
/* some important defines:
*
* WORDS_ALIGNED : if defined, the host cpu can only make word aligned
diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
index de8a7200a9..9fb1d541d4 100644
--- a/include/exec/cpu-common.h
+++ b/include/exec/cpu-common.h
@@ -13,6 +13,8 @@
#include "qemu/bswap.h"
#include "qemu/queue.h"
+#include "qemu/fprintf-fn.h"
+#include "qemu/typedefs.h"
/**
* CPUListState:
diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h
index d5aecaf49e..98b9cff310 100644
--- a/include/exec/cpu-defs.h
+++ b/include/exec/cpu-defs.h
@@ -56,20 +56,6 @@ typedef uint64_t target_ulong;
#error TARGET_LONG_SIZE undefined
#endif
-#define EXCP_INTERRUPT 0x10000 /* async interruption */
-#define EXCP_HLT 0x10001 /* hlt instruction reached */
-#define EXCP_DEBUG 0x10002 /* cpu stopped after a breakpoint or singlestep */
-#define EXCP_HALTED 0x10003 /* cpu is halted (waiting for external event) */
-#define EXCP_YIELD 0x10004 /* cpu wants to yield timeslice to another */
-
-/* Only the bottom TB_JMP_PAGE_BITS of the jump cache hash bits vary for
- addresses on the same page. The top bits are the same. This allows
- TLB invalidation to quickly clear a subset of the hash table. */
-#define TB_JMP_PAGE_BITS (TB_JMP_CACHE_BITS / 2)
-#define TB_JMP_PAGE_SIZE (1 << TB_JMP_PAGE_BITS)
-#define TB_JMP_ADDR_MASK (TB_JMP_PAGE_SIZE - 1)
-#define TB_JMP_PAGE_MASK (TB_JMP_CACHE_SIZE - TB_JMP_PAGE_SIZE)
-
#if !defined(CONFIG_USER_ONLY)
/* use a fully associative victim tlb of 8 entries */
#define CPU_VTLB_SIZE 8
@@ -161,7 +147,6 @@ typedef struct CPUIOTLBEntry {
#endif
-#define CPU_TEMP_BUF_NLONGS 128
#define CPU_COMMON \
/* soft mmu support */ \
CPU_COMMON_TLB \
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 2573e8c36e..d678114cb2 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -195,26 +195,6 @@ struct TBContext {
int tb_invalidated_flag;
};
-static inline unsigned int tb_jmp_cache_hash_page(target_ulong pc)
-{
- target_ulong tmp;
- tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS));
- return (tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK;
-}
-
-static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc)
-{
- target_ulong tmp;
- tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS));
- return (((tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK)
- | (tmp & TB_JMP_ADDR_MASK));
-}
-
-static inline unsigned int tb_phys_hash_func(tb_page_addr_t pc)
-{
- return (pc >> 2) & (CODE_GEN_PHYS_HASH_SIZE - 1);
-}
-
void tb_free(TranslationBlock *tb);
void tb_flush(CPUArchState *env);
void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
diff --git a/include/exec/memory.h b/include/exec/memory.h
index 8ae004eb06..139471500f 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -23,7 +23,6 @@
#include <stdint.h>
#include <stdbool.h>
-#include "qemu-common.h"
#include "exec/cpu-common.h"
#ifndef CONFIG_USER_ONLY
#include "exec/hwaddr.h"
@@ -180,6 +179,7 @@ struct MemoryRegion {
bool rom_device;
bool warning_printed; /* For reservations */
bool flush_coalesced_mmio;
+ bool global_locking;
MemoryRegion *alias;
hwaddr alias_offset;
int32_t priority;
@@ -825,6 +825,31 @@ void memory_region_set_flush_coalesced(MemoryRegion *mr);
void memory_region_clear_flush_coalesced(MemoryRegion *mr);
/**
+ * memory_region_set_global_locking: Declares the access processing requires
+ * QEMU's global lock.
+ *
+ * When this is invoked, accesses to the memory region will be processed while
+ * holding the global lock of QEMU. This is the default behavior of memory
+ * regions.
+ *
+ * @mr: the memory region to be updated.
+ */
+void memory_region_set_global_locking(MemoryRegion *mr);
+
+/**
+ * memory_region_clear_global_locking: Declares that access processing does
+ * not depend on the QEMU global lock.
+ *
+ * By clearing this property, accesses to the memory region will be processed
+ * outside of QEMU's global lock (unless the lock is held on when issuing the
+ * access request). In this case, the device model implementing the access
+ * handlers is responsible for synchronization of concurrency.
+ *
+ * @mr: the memory region to be updated.
+ */
+void memory_region_clear_global_locking(MemoryRegion *mr);
+
+/**
* memory_region_add_eventfd: Request an eventfd to be triggered when a word
* is written to a location.
*
diff --git a/include/exec/tb-hash.h b/include/exec/tb-hash.h
new file mode 100644
index 0000000000..0f4e8a08af
--- /dev/null
+++ b/include/exec/tb-hash.h
@@ -0,0 +1,51 @@
+/*
+ * internal execution defines for qemu
+ *
+ * Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef EXEC_TB_HASH
+#define EXEC_TB_HASH
+
+/* Only the bottom TB_JMP_PAGE_BITS of the jump cache hash bits vary for
+ addresses on the same page. The top bits are the same. This allows
+ TLB invalidation to quickly clear a subset of the hash table. */
+#define TB_JMP_PAGE_BITS (TB_JMP_CACHE_BITS / 2)
+#define TB_JMP_PAGE_SIZE (1 << TB_JMP_PAGE_BITS)
+#define TB_JMP_ADDR_MASK (TB_JMP_PAGE_SIZE - 1)
+#define TB_JMP_PAGE_MASK (TB_JMP_CACHE_SIZE - TB_JMP_PAGE_SIZE)
+
+static inline unsigned int tb_jmp_cache_hash_page(target_ulong pc)
+{
+ target_ulong tmp;
+ tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS));
+ return (tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK;
+}
+
+static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc)
+{
+ target_ulong tmp;
+ tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS));
+ return (((tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK)
+ | (tmp & TB_JMP_ADDR_MASK));
+}
+
+static inline unsigned int tb_phys_hash_func(tb_page_addr_t pc)
+{
+ return (pc >> 2) & (CODE_GEN_PHYS_HASH_SIZE - 1);
+}
+
+#endif
diff --git a/include/hw/arm/arm.h b/include/hw/arm/arm.h
index 760804cc46..4dcd4f9b63 100644
--- a/include/hw/arm/arm.h
+++ b/include/hw/arm/arm.h
@@ -14,6 +14,7 @@
#include "exec/memory.h"
#include "hw/irq.h"
#include "qemu/notify.h"
+#include "cpu.h"
/* armv7m.c */
qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
diff --git a/include/qemu-common.h b/include/qemu-common.h
index d52d09cfb8..237d6547b3 100644
--- a/include/qemu-common.h
+++ b/include/qemu-common.h
@@ -15,6 +15,7 @@
#include "qemu/compiler.h"
#include "config-host.h"
#include "qemu/typedefs.h"
+#include "qemu/fprintf-fn.h"
#if defined(__arm__) || defined(__sparc__) || defined(__mips__) || defined(__hppa__) || defined(__ia64__)
#define WORDS_ALIGNED
@@ -85,9 +86,6 @@
# error Unknown pointer size
#endif
-typedef int (*fprintf_function)(FILE *f, const char *fmt, ...)
- GCC_FMT_ATTR(2, 3);
-
#ifdef _WIN32
#define fsync _commit
#if !defined(lseek)
@@ -455,6 +453,7 @@ void qemu_hexdump(const char *buf, FILE *fp, const char *prefix, size_t size);
#define VECTYPE __vector unsigned char
#define SPLAT(p) vec_splat(vec_ld(0, p), 0)
#define ALL_EQ(v1, v2) vec_all_eq(v1, v2)
+#define VEC_OR(v1, v2) ((v1) | (v2))
/* altivec.h may redefine the bool macro as vector type.
* Reset it to POSIX semantics. */
#define bool _Bool
@@ -463,10 +462,12 @@ void qemu_hexdump(const char *buf, FILE *fp, const char *prefix, size_t size);
#define VECTYPE __m128i
#define SPLAT(p) _mm_set1_epi8(*(p))
#define ALL_EQ(v1, v2) (_mm_movemask_epi8(_mm_cmpeq_epi8(v1, v2)) == 0xFFFF)
+#define VEC_OR(v1, v2) (_mm_or_si128(v1, v2))
#else
#define VECTYPE unsigned long
#define SPLAT(p) (*(p) * (~0UL / 255))
#define ALL_EQ(v1, v2) ((v1) == (v2))
+#define VEC_OR(v1, v2) ((v1) | (v2))
#endif
#define BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR 8
diff --git a/include/qemu/fprintf-fn.h b/include/qemu/fprintf-fn.h
new file mode 100644
index 0000000000..9ddc90f1c5
--- /dev/null
+++ b/include/qemu/fprintf-fn.h
@@ -0,0 +1,17 @@
+/*
+ * Typedef for fprintf-alike function pointers.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_FPRINTF_FN_H
+#define QEMU_FPRINTF_FN_H 1
+
+#include "qemu/compiler.h"
+#include <stdio.h>
+
+typedef int (*fprintf_function)(FILE *f, const char *fmt, ...)
+ GCC_FMT_ATTR(2, 3);
+
+#endif
diff --git a/include/qemu/main-loop.h b/include/qemu/main-loop.h
index 0f4a0fd4b2..bc18ca30e4 100644
--- a/include/qemu/main-loop.h
+++ b/include/qemu/main-loop.h
@@ -223,6 +223,16 @@ int qemu_add_child_watch(pid_t pid);
#endif
/**
+ * qemu_mutex_iothread_locked: Return lock status of the main loop mutex.
+ *
+ * The main loop mutex is the coarsest lock in QEMU, and as such it
+ * must always be taken outside other locks. This function helps
+ * functions take different paths depending on whether the current
+ * thread is running within the main loop mutex.
+ */
+bool qemu_mutex_iothread_locked(void);
+
+/**
* qemu_mutex_lock_iothread: Lock the main loop mutex.
*
* This function locks the main loop mutex. The mutex is taken by
diff --git a/kvm-all.c b/kvm-all.c
index 53e01d468e..df57da0bf2 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -1099,9 +1099,17 @@ static int kvm_irqchip_get_virq(KVMState *s)
uint32_t *word = s->used_gsi_bitmap;
int max_words = ALIGN(s->gsi_count, 32) / 32;
int i, zeroes;
- bool retry = true;
-again:
+ /*
+ * PIC and IOAPIC share the first 16 GSI numbers, thus the available
+ * GSI numbers are more than the number of IRQ route. Allocating a GSI
+ * number can succeed even though a new route entry cannot be added.
+ * When this happens, flush dynamic MSI entries to free IRQ route entries.
+ */
+ if (!s->direct_msi && s->irq_routes->nr == s->gsi_count) {
+ kvm_flush_dynamic_msi_routes(s);
+ }
+
/* Return the lowest unused GSI in the bitmap */
for (i = 0; i < max_words; i++) {
zeroes = ctz32(~word[i]);
@@ -1111,11 +1119,6 @@ again:
return zeroes + i * 32;
}
- if (!s->direct_msi && retry) {
- retry = false;
- kvm_flush_dynamic_msi_routes(s);
- goto again;
- }
return -ENOSPC;
}
@@ -1752,6 +1755,8 @@ int kvm_cpu_exec(CPUState *cpu)
return EXCP_HLT;
}
+ qemu_mutex_unlock_iothread();
+
do {
MemTxAttrs attrs;
@@ -1770,11 +1775,9 @@ int kvm_cpu_exec(CPUState *cpu)
*/
qemu_cpu_kick_self();
}
- qemu_mutex_unlock_iothread();
run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
- qemu_mutex_lock_iothread();
attrs = kvm_arch_post_run(cpu, run);
if (run_ret < 0) {
@@ -1801,6 +1804,7 @@ int kvm_cpu_exec(CPUState *cpu)
switch (run->exit_reason) {
case KVM_EXIT_IO:
DPRINTF("handle_io\n");
+ /* Called outside BQL */
kvm_handle_io(run->io.port, attrs,
(uint8_t *)run + run->io.data_offset,
run->io.direction,
@@ -1810,6 +1814,7 @@ int kvm_cpu_exec(CPUState *cpu)
break;
case KVM_EXIT_MMIO:
DPRINTF("handle_mmio\n");
+ /* Called outside BQL */
address_space_rw(&address_space_memory,
run->mmio.phys_addr, attrs,
run->mmio.data,
@@ -1857,6 +1862,8 @@ int kvm_cpu_exec(CPUState *cpu)
}
} while (ret == 0);
+ qemu_mutex_lock_iothread();
+
if (ret < 0) {
cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
vm_stop(RUN_STATE_INTERNAL_ERROR);
diff --git a/memory.c b/memory.c
index 3ac0bd20d2..5a0cc66982 100644
--- a/memory.c
+++ b/memory.c
@@ -396,9 +396,6 @@ static MemTxResult memory_region_read_accessor(MemoryRegion *mr,
{
uint64_t tmp;
- if (mr->flush_coalesced_mmio) {
- qemu_flush_coalesced_mmio_buffer();
- }
tmp = mr->ops->read(mr->opaque, addr, size);
trace_memory_region_ops_read(mr, addr, tmp, size);
*value |= (tmp & mask) << shift;
@@ -416,9 +413,6 @@ static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,
uint64_t tmp = 0;
MemTxResult r;
- if (mr->flush_coalesced_mmio) {
- qemu_flush_coalesced_mmio_buffer();
- }
r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs);
trace_memory_region_ops_read(mr, addr, tmp, size);
*value |= (tmp & mask) << shift;
@@ -451,9 +445,6 @@ static MemTxResult memory_region_write_accessor(MemoryRegion *mr,
{
uint64_t tmp;
- if (mr->flush_coalesced_mmio) {
- qemu_flush_coalesced_mmio_buffer();
- }
tmp = (*value >> shift) & mask;
trace_memory_region_ops_write(mr, addr, tmp, size);
mr->ops->write(mr->opaque, addr, tmp, size);
@@ -470,9 +461,6 @@ static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,
{
uint64_t tmp;
- if (mr->flush_coalesced_mmio) {
- qemu_flush_coalesced_mmio_buffer();
- }
tmp = (*value >> shift) & mask;
trace_memory_region_ops_write(mr, addr, tmp, size);
return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs);
@@ -1012,6 +1000,7 @@ static void memory_region_initfn(Object *obj)
mr->ram_addr = RAM_ADDR_INVALID;
mr->enabled = true;
mr->romd_mode = true;
+ mr->global_locking = true;
mr->destructor = memory_region_destructor_none;
QTAILQ_INIT(&mr->subregions);
QTAILQ_INIT(&mr->coalesced);
@@ -1646,6 +1635,16 @@ void memory_region_clear_flush_coalesced(MemoryRegion *mr)
}
}
+void memory_region_set_global_locking(MemoryRegion *mr)
+{
+ mr->global_locking = true;
+}
+
+void memory_region_clear_global_locking(MemoryRegion *mr)
+{
+ mr->global_locking = false;
+}
+
void memory_region_add_eventfd(MemoryRegion *mr,
hwaddr addr,
unsigned size,
diff --git a/memory_mapping.c b/memory_mapping.c
index 7b69801cb8..36d6b26046 100644
--- a/memory_mapping.c
+++ b/memory_mapping.c
@@ -13,8 +13,8 @@
#include <glib.h>
+#include "qemu-common.h"
#include "cpu.h"
-#include "exec/cpu-all.h"
#include "sysemu/memory_mapping.h"
#include "exec/memory.h"
#include "exec/address-spaces.h"
diff --git a/stubs/iothread-lock.c b/stubs/iothread-lock.c
index 5d8aca1b37..dda6f6b58d 100644
--- a/stubs/iothread-lock.c
+++ b/stubs/iothread-lock.c
@@ -1,6 +1,11 @@
#include "qemu-common.h"
#include "qemu/main-loop.h"
+bool qemu_mutex_iothread_locked(void)
+{
+ return true;
+}
+
void qemu_mutex_lock_iothread(void)
{
}
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index daced5cb94..6426600c63 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -2191,7 +2191,10 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
/* Inject NMI */
if (cpu->interrupt_request & CPU_INTERRUPT_NMI) {
+ qemu_mutex_lock_iothread();
cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
+ qemu_mutex_unlock_iothread();
+
DPRINTF("injected NMI\n");
ret = kvm_vcpu_ioctl(cpu, KVM_NMI);
if (ret < 0) {
@@ -2200,6 +2203,10 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
}
}
+ if (!kvm_irqchip_in_kernel()) {
+ qemu_mutex_lock_iothread();
+ }
+
/* Force the VCPU out of its inner loop to process any INIT requests
* or (for userspace APIC, but it is cheap to combine the checks here)
* pending TPR access reports.
@@ -2243,6 +2250,8 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
DPRINTF("setting tpr\n");
run->cr8 = cpu_get_apic_tpr(x86_cpu->apic_state);
+
+ qemu_mutex_unlock_iothread();
}
}
@@ -2256,8 +2265,17 @@ MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
} else {
env->eflags &= ~IF_MASK;
}
+
+ /* We need to protect the apic state against concurrent accesses from
+ * different threads in case the userspace irqchip is used. */
+ if (!kvm_irqchip_in_kernel()) {
+ qemu_mutex_lock_iothread();
+ }
cpu_set_apic_tpr(x86_cpu->apic_state, run->cr8);
cpu_set_apic_base(x86_cpu->apic_state, run->apic_base);
+ if (!kvm_irqchip_in_kernel()) {
+ qemu_mutex_unlock_iothread();
+ }
return cpu_get_mem_attrs(env);
}
@@ -2550,13 +2568,17 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
switch (run->exit_reason) {
case KVM_EXIT_HLT:
DPRINTF("handle_hlt\n");
+ qemu_mutex_lock_iothread();
ret = kvm_handle_halt(cpu);
+ qemu_mutex_unlock_iothread();
break;
case KVM_EXIT_SET_TPR:
ret = 0;
break;
case KVM_EXIT_TPR_ACCESS:
+ qemu_mutex_lock_iothread();
ret = kvm_handle_tpr_access(cpu);
+ qemu_mutex_unlock_iothread();
break;
case KVM_EXIT_FAIL_ENTRY:
code = run->fail_entry.hardware_entry_failure_reason;
@@ -2582,7 +2604,9 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
break;
case KVM_EXIT_DEBUG:
DPRINTF("kvm_exit_debug\n");
+ qemu_mutex_lock_iothread();
ret = kvm_handle_debug(cpu, &run->debug.arch);
+ qemu_mutex_unlock_iothread();
break;
default:
fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
diff --git a/target-mips/kvm.c b/target-mips/kvm.c
index 948619fbab..7d2293d934 100644
--- a/target-mips/kvm.c
+++ b/target-mips/kvm.c
@@ -99,6 +99,8 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
int r;
struct kvm_mips_interrupt intr;
+ qemu_mutex_lock_iothread();
+
if ((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
cpu_mips_io_interrupts_pending(cpu)) {
intr.cpu = -1;
@@ -109,6 +111,8 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
__func__, cs->cpu_index, intr.irq);
}
}
+
+ qemu_mutex_unlock_iothread();
}
MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
index afb4696b8a..ddf469fe09 100644
--- a/target-ppc/kvm.c
+++ b/target-ppc/kvm.c
@@ -1242,6 +1242,8 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
int r;
unsigned irq;
+ qemu_mutex_lock_iothread();
+
/* PowerPC QEMU tracks the various core input pins (interrupt, critical
* interrupt, reset, etc) in PPC-specific env->irq_input_state. */
if (!cap_interrupt_level &&
@@ -1269,6 +1271,8 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
/* We don't know if there are more interrupts pending after this. However,
* the guest will return to userspace in the course of handling this one
* anyways, so we will get a chance to deliver the rest. */
+
+ qemu_mutex_unlock_iothread();
}
MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
@@ -1570,6 +1574,8 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
CPUPPCState *env = &cpu->env;
int ret;
+ qemu_mutex_lock_iothread();
+
switch (run->exit_reason) {
case KVM_EXIT_DCR:
if (run->dcr.is_write) {
@@ -1620,6 +1626,7 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
break;
}
+ qemu_mutex_unlock_iothread();
return ret;
}
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 135111a2c4..ae3a0affec 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -2007,6 +2007,8 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
S390CPU *cpu = S390_CPU(cs);
int ret = 0;
+ qemu_mutex_lock_iothread();
+
switch (run->exit_reason) {
case KVM_EXIT_S390_SIEIC:
ret = handle_intercept(cpu);
@@ -2027,6 +2029,7 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
fprintf(stderr, "Unknown KVM exit: %d\n", run->exit_reason);
break;
}
+ qemu_mutex_unlock_iothread();
if (ret == 0) {
ret = EXCP_INTERRUPT;
diff --git a/target-s390x/mmu_helper.c b/target-s390x/mmu_helper.c
index 815ff42dde..1ea6d812c2 100644
--- a/target-s390x/mmu_helper.c
+++ b/target-s390x/mmu_helper.c
@@ -17,8 +17,8 @@
#include "qemu/error-report.h"
#include "exec/address-spaces.h"
-#include "sysemu/kvm.h"
#include "cpu.h"
+#include "sysemu/kvm.h"
/* #define DEBUG_S390 */
/* #define DEBUG_S390_PTE */
diff --git a/tcg/tcg.h b/tcg/tcg.h
index 41e486959d..231a781524 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -29,6 +29,8 @@
#include "qemu/bitops.h"
#include "tcg-target.h"
+#define CPU_TEMP_BUF_NLONGS 128
+
/* Default target word size to pointer size. */
#ifndef TCG_TARGET_REG_BITS
# if UINTPTR_MAX == UINT32_MAX
diff --git a/translate-all.c b/translate-all.c
index b6b0e1c098..412bc9005f 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -58,6 +58,7 @@
#endif
#include "exec/cputlb.h"
+#include "exec/tb-hash.h"
#include "translate-all.h"
#include "qemu/bitmap.h"
#include "qemu/timer.h"
diff --git a/util/cutils.c b/util/cutils.c
index 144b25c05a..5d1c9ebe05 100644
--- a/util/cutils.c
+++ b/util/cutils.c
@@ -207,13 +207,13 @@ size_t buffer_find_nonzero_offset(const void *buf, size_t len)
for (i = BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR;
i < len / sizeof(VECTYPE);
i += BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR) {
- VECTYPE tmp0 = p[i + 0] | p[i + 1];
- VECTYPE tmp1 = p[i + 2] | p[i + 3];
- VECTYPE tmp2 = p[i + 4] | p[i + 5];
- VECTYPE tmp3 = p[i + 6] | p[i + 7];
- VECTYPE tmp01 = tmp0 | tmp1;
- VECTYPE tmp23 = tmp2 | tmp3;
- if (!ALL_EQ(tmp01 | tmp23, zero)) {
+ VECTYPE tmp0 = VEC_OR(p[i + 0], p[i + 1]);
+ VECTYPE tmp1 = VEC_OR(p[i + 2], p[i + 3]);
+ VECTYPE tmp2 = VEC_OR(p[i + 4], p[i + 5]);
+ VECTYPE tmp3 = VEC_OR(p[i + 6], p[i + 7]);
+ VECTYPE tmp01 = VEC_OR(tmp0, tmp1);
+ VECTYPE tmp23 = VEC_OR(tmp2, tmp3);
+ if (!ALL_EQ(VEC_OR(tmp01, tmp23), zero)) {
break;
}
}