diff options
Diffstat (limited to 'include/exec/ram_addr.h')
| -rw-r--r-- | include/exec/ram_addr.h | 332 |
1 files changed, 204 insertions, 128 deletions
diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 3abb639056..de45ba7bc9 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -20,36 +20,61 @@ #define RAM_ADDR_H #ifndef CONFIG_USER_ONLY -#include "hw/xen/xen.h" +#include "cpu.h" +#include "sysemu/xen.h" +#include "sysemu/tcg.h" #include "exec/ramlist.h" +#include "exec/ramblock.h" +#include "exec/exec-all.h" -struct RAMBlock { - struct rcu_head rcu; - struct MemoryRegion *mr; - uint8_t *host; - ram_addr_t offset; - ram_addr_t used_length; - ram_addr_t max_length; - void (*resized)(const char*, uint64_t length, void *host); - uint32_t flags; - /* Protected by iothread lock. */ - char idstr[256]; - /* RCU-enabled, writes protected by the ramlist lock */ - QLIST_ENTRY(RAMBlock) next; - QLIST_HEAD(, RAMBlockNotifier) ramblock_notifiers; - int fd; - size_t page_size; - /* dirty bitmap used during migration */ - unsigned long *bmap; - /* bitmap of pages that haven't been sent even once - * only maintained and used in postcopy at the moment - * where it's used to send the dirtymap at the start - * of the postcopy phase - */ - unsigned long *unsentmap; - /* bitmap of already received pages in postcopy */ - unsigned long *receivedmap; -}; +extern uint64_t total_dirty_pages; + +/** + * clear_bmap_size: calculate clear bitmap size + * + * @pages: number of guest pages + * @shift: guest page number shift + * + * Returns: number of bits for the clear bitmap + */ +static inline long clear_bmap_size(uint64_t pages, uint8_t shift) +{ + return DIV_ROUND_UP(pages, 1UL << shift); +} + +/** + * clear_bmap_set: set clear bitmap for the page range. Must be with + * bitmap_mutex held. + * + * @rb: the ramblock to operate on + * @start: the start page number + * @size: number of pages to set in the bitmap + * + * Returns: None + */ +static inline void clear_bmap_set(RAMBlock *rb, uint64_t start, + uint64_t npages) +{ + uint8_t shift = rb->clear_bmap_shift; + + bitmap_set(rb->clear_bmap, start >> shift, clear_bmap_size(npages, shift)); +} + +/** + * clear_bmap_test_and_clear: test clear bitmap for the page, clear if set. + * Must be with bitmap_mutex held. + * + * @rb: the ramblock to operate on + * @page: the page number to check + * + * Returns: true if the bit was set, false otherwise + */ +static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page) +{ + uint8_t shift = rb->clear_bmap_shift; + + return bitmap_test_and_clear(rb->clear_bmap, page >> shift, 1); +} static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) { @@ -72,7 +97,8 @@ static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr, bool ramblock_is_pmem(RAMBlock *rb); -long qemu_getrampagesize(void); +long qemu_minrampagesize(void); +long qemu_maxrampagesize(void); /** * qemu_ram_alloc_from_file, @@ -82,12 +108,11 @@ long qemu_getrampagesize(void); * Parameters: * @size: the size in bytes of the ram block * @mr: the memory region where the ram block is - * @ram_flags: specify the properties of the ram block, which can be one - * or bit-or of following values - * - RAM_SHARED: mmap the backing file or device with MAP_SHARED - * - RAM_PMEM: the backend @mem_path or @fd is persistent memory - * Other bits are ignored. + * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM, + * RAM_NORESERVE, RAM_PROTECTED, RAM_NAMED_FILE, RAM_READONLY, + * RAM_READONLY_FD * @mem_path or @fd: specify the backing file or device + * @offset: Offset into target file * @errp: pointer to Error*, to store an error if it happens * * Return: @@ -96,14 +121,14 @@ long qemu_getrampagesize(void); */ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, const char *mem_path, - Error **errp); + off_t offset, Error **errp); RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, - uint32_t ram_flags, int fd, + uint32_t ram_flags, int fd, off_t offset, Error **errp); RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, MemoryRegion *mr, Error **errp); -RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share, MemoryRegion *mr, +RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags, MemoryRegion *mr, Error **errp); RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size, void (*resized)(const char*, @@ -114,11 +139,17 @@ void qemu_ram_free(RAMBlock *block); int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp); +void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length); + +/* Clear whole block of mem */ +static inline void qemu_ram_block_writeback(RAMBlock *block) +{ + qemu_ram_msync(block, 0, block->used_length); +} + #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) -void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end); - static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, ram_addr_t length, unsigned client) @@ -133,30 +164,29 @@ static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; page = start >> TARGET_PAGE_BITS; - rcu_read_lock(); - - blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); + WITH_RCU_READ_LOCK_GUARD() { + blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); + + idx = page / DIRTY_MEMORY_BLOCK_SIZE; + offset = page % DIRTY_MEMORY_BLOCK_SIZE; + base = page - offset; + while (page < end) { + unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); + unsigned long num = next - base; + unsigned long found = find_next_bit(blocks->blocks[idx], + num, offset); + if (found < num) { + dirty = true; + break; + } - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - unsigned long num = next - base; - unsigned long found = find_next_bit(blocks->blocks[idx], num, offset); - if (found < num) { - dirty = true; - break; + page = next; + idx++; + offset = 0; + base += DIRTY_MEMORY_BLOCK_SIZE; } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; } - rcu_read_unlock(); - return dirty; } @@ -174,9 +204,9 @@ static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; page = start >> TARGET_PAGE_BITS; - rcu_read_lock(); + RCU_READ_LOCK_GUARD(); - blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); + blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); idx = page / DIRTY_MEMORY_BLOCK_SIZE; offset = page % DIRTY_MEMORY_BLOCK_SIZE; @@ -196,8 +226,6 @@ static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, base += DIRTY_MEMORY_BLOCK_SIZE; } - rcu_read_unlock(); - return dirty; } @@ -249,13 +277,11 @@ static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, idx = page / DIRTY_MEMORY_BLOCK_SIZE; offset = page % DIRTY_MEMORY_BLOCK_SIZE; - rcu_read_lock(); + RCU_READ_LOCK_GUARD(); - blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); + blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); set_bit_atomic(offset, blocks->blocks[idx]); - - rcu_read_unlock(); } static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, @@ -274,53 +300,60 @@ static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; page = start >> TARGET_PAGE_BITS; - rcu_read_lock(); + WITH_RCU_READ_LOCK_GUARD() { + for (i = 0; i < DIRTY_MEMORY_NUM; i++) { + blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]); + } - for (i = 0; i < DIRTY_MEMORY_NUM; i++) { - blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]); - } + idx = page / DIRTY_MEMORY_BLOCK_SIZE; + offset = page % DIRTY_MEMORY_BLOCK_SIZE; + base = page - offset; + while (page < end) { + unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); + if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { + bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], + offset, next - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { + bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], + offset, next - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { + bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], + offset, next - page); + } - if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], - offset, next - page); - } - if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], - offset, next - page); - } - if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], - offset, next - page); + page = next; + idx++; + offset = 0; + base += DIRTY_MEMORY_BLOCK_SIZE; } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; } - rcu_read_unlock(); - xen_hvm_modified_memory(start, length); } #if !defined(_WIN32) -static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, - ram_addr_t start, - ram_addr_t pages) + +/* + * Contrary to cpu_physical_memory_sync_dirty_bitmap() this function returns + * the number of dirty pages in @bitmap passed as argument. On the other hand, + * cpu_physical_memory_sync_dirty_bitmap() returns newly dirtied pages that + * weren't set in the global migration bitmap. + */ +static inline +uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, + ram_addr_t start, + ram_addr_t pages) { unsigned long i, j; - unsigned long page_number, c; + unsigned long page_number, c, nbits; hwaddr addr; ram_addr_t ram_addr; + uint64_t num_dirty = 0; unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; - unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE; + unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE; unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); /* start address is aligned at the start of a word? */ @@ -336,34 +369,52 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, offset = BIT_WORD((start >> TARGET_PAGE_BITS) % DIRTY_MEMORY_BLOCK_SIZE); - rcu_read_lock(); - - for (i = 0; i < DIRTY_MEMORY_NUM; i++) { - blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks; - } - - for (k = 0; k < nr; k++) { - if (bitmap[k]) { - unsigned long temp = leul_to_cpu(bitmap[k]); + WITH_RCU_READ_LOCK_GUARD() { + for (i = 0; i < DIRTY_MEMORY_NUM; i++) { + blocks[i] = + qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks; + } - atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp); - atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); - if (tcg_enabled()) { - atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp); + for (k = 0; k < nr; k++) { + if (bitmap[k]) { + unsigned long temp = leul_to_cpu(bitmap[k]); + + nbits = ctpopl(temp); + qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); + + if (global_dirty_tracking) { + qatomic_or( + &blocks[DIRTY_MEMORY_MIGRATION][idx][offset], + temp); + if (unlikely( + global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { + total_dirty_pages += nbits; + } + } + + num_dirty += nbits; + + if (tcg_enabled()) { + qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], + temp); + } } - } - if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { - offset = 0; - idx++; + if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { + offset = 0; + idx++; + } } } - rcu_read_unlock(); - xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); } else { uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; + + if (!global_dirty_tracking) { + clients &= ~(1 << DIRTY_MEMORY_MIGRATION); + } + /* * bitmap-traveling is faster than memory-traveling (for addr...) * especially when most of the memory is not dirty. @@ -371,6 +422,11 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, for (i = 0; i < len; i++) { if (bitmap[i] != 0) { c = leul_to_cpu(bitmap[i]); + nbits = ctpopl(c); + if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { + total_dirty_pages += nbits; + } + num_dirty += nbits; do { j = ctzl(c); c &= ~(1ul << j); @@ -383,15 +439,25 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, } } } + + return num_dirty; } #endif /* not _WIN32 */ +static inline void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, + ram_addr_t length) +{ + if (tcg_enabled()) { + tlb_reset_dirty_range_all(start, length); + } + +} bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, unsigned client); DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty - (ram_addr_t start, ram_addr_t length, unsigned client); + (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client); bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, ram_addr_t start, @@ -406,11 +472,11 @@ static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, } +/* Called with RCU critical section */ static inline uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, ram_addr_t start, - ram_addr_t length, - uint64_t *real_dirty_pages) + ram_addr_t length) { ram_addr_t addr; unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS); @@ -429,16 +495,13 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, DIRTY_MEMORY_BLOCK_SIZE); unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); - rcu_read_lock(); - - src = atomic_rcu_read( + src = qatomic_rcu_read( &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; for (k = page; k < page + nr; k++) { if (src[idx][offset]) { - unsigned long bits = atomic_xchg(&src[idx][offset], 0); + unsigned long bits = qatomic_xchg(&src[idx][offset], 0); unsigned long new_dirty; - *real_dirty_pages += ctpopl(bits); new_dirty = ~dest[k]; dest[k] |= bits; new_dirty &= bits; @@ -450,8 +513,22 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, idx++; } } + if (num_dirty) { + cpu_physical_memory_dirty_bits_cleared(start, length); + } - rcu_read_unlock(); + if (rb->clear_bmap) { + /* + * Postpone the dirty bitmap clear to the point before we + * really send the pages, also we will split the clear + * dirty procedure into smaller chunks. + */ + clear_bmap_set(rb, start >> TARGET_PAGE_BITS, + length >> TARGET_PAGE_BITS); + } else { + /* Slow path - still do that in a huge chunk */ + memory_region_clear_dirty_bitmap(rb->mr, start, length); + } } else { ram_addr_t offset = rb->offset; @@ -460,7 +537,6 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, start + addr + offset, TARGET_PAGE_SIZE, DIRTY_MEMORY_MIGRATION)) { - *real_dirty_pages += 1; long k = (start + addr) >> TARGET_PAGE_BITS; if (!test_and_set_bit(k, dest)) { num_dirty++; |