diff options
Diffstat (limited to 'drivers/gpu/drm/i915/gt/selftest_lrc.c')
| -rw-r--r-- | drivers/gpu/drm/i915/gt/selftest_lrc.c | 2923 |
1 files changed, 1495 insertions, 1428 deletions
diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 401e8b539297..8b2c11dbe354 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -1,1835 +1,1902 @@ +// SPDX-License-Identifier: MIT /* - * SPDX-License-Identifier: MIT - * * Copyright © 2018 Intel Corporation */ #include <linux/prime_numbers.h> -#include "gem/i915_gem_pm.h" -#include "gt/intel_reset.h" +#include "gem/i915_gem_internal.h" #include "i915_selftest.h" +#include "intel_engine_heartbeat.h" +#include "intel_engine_pm.h" +#include "intel_reset.h" +#include "intel_ring.h" +#include "selftest_engine_heartbeat.h" #include "selftests/i915_random.h" #include "selftests/igt_flush_test.h" #include "selftests/igt_live_test.h" #include "selftests/igt_spinner.h" #include "selftests/lib_sw_fence.h" +#include "shmem_utils.h" #include "gem/selftests/igt_gem_utils.h" #include "gem/selftests/mock_context.h" -static int live_sanitycheck(void *arg) +#define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4) +#define NUM_GPR 16 +#define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ + +static struct i915_vma *create_scratch(struct intel_gt *gt) { - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct i915_gem_context *ctx; - enum intel_engine_id id; - struct igt_spinner spin; - intel_wakeref_t wakeref; - int err = -ENOMEM; + return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE); +} - if (!HAS_LOGICAL_RING_CONTEXTS(i915)) - return 0; +static bool is_active(struct i915_request *rq) +{ + if (i915_request_is_active(rq)) + return true; - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(&i915->runtime_pm); + if (i915_request_on_hold(rq)) + return true; - if (igt_spinner_init(&spin, i915)) - goto err_unlock; + if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq)) + return true; - ctx = kernel_context(i915); - if (!ctx) - goto err_spin; + return false; +} - for_each_engine(engine, i915, id) { - struct i915_request *rq; +static int wait_for_submit(struct intel_engine_cs *engine, + struct i915_request *rq, + unsigned long timeout) +{ + /* Ignore our own attempts to suppress excess tasklets */ + tasklet_hi_schedule(&engine->sched_engine->tasklet); - rq = igt_spinner_create_request(&spin, ctx, engine, MI_NOOP); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_ctx; - } + timeout += jiffies; + do { + bool done = time_after(jiffies, timeout); - i915_request_add(rq); - if (!igt_wait_for_spinner(&spin, rq)) { - GEM_TRACE("spinner failed to start\n"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx; - } + if (i915_request_completed(rq)) /* that was quick! */ + return 0; - igt_spinner_end(&spin); - if (igt_flush_test(i915, I915_WAIT_LOCKED)) { - err = -EIO; - goto err_ctx; - } - } + /* Wait until the HW has acknowleged the submission (or err) */ + intel_engine_flush_submission(engine); + if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) + return 0; - err = 0; -err_ctx: - kernel_context_close(ctx); -err_spin: - igt_spinner_fini(&spin); -err_unlock: - igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(&i915->runtime_pm, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; + if (done) + return -ETIME; + + cond_resched(); + } while (1); } -static int live_busywait_preempt(void *arg) +static int emit_semaphore_signal(struct intel_context *ce, void *slot) { - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx_hi, *ctx_lo; - struct intel_engine_cs *engine; - struct drm_i915_gem_object *obj; - struct i915_vma *vma; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - u32 *map; + const u32 offset = + i915_ggtt_offset(ce->engine->status_page.vma) + + offset_in_page(slot); + struct i915_request *rq; + u32 *cs; - /* - * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can - * preempt the busywaits used to synchronise between rings. - */ + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return PTR_ERR(rq); - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(&i915->runtime_pm); + cs = intel_ring_begin(rq, 4); + if (IS_ERR(cs)) { + i915_request_add(rq); + return PTR_ERR(cs); + } - ctx_hi = kernel_context(i915); - if (!ctx_hi) - goto err_unlock; - ctx_hi->sched.priority = - I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = offset; + *cs++ = 0; + *cs++ = 1; - ctx_lo = kernel_context(i915); - if (!ctx_lo) - goto err_ctx_hi; - ctx_lo->sched.priority = - I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); + intel_ring_advance(rq, cs); - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto err_ctx_lo; - } + rq->sched.attr.priority = I915_PRIORITY_BARRIER; + i915_request_add(rq); + return 0; +} + +static int context_flush(struct intel_context *ce, long timeout) +{ + struct i915_request *rq; + struct dma_fence *fence; + int err = 0; + + rq = intel_engine_create_kernel_request(ce->engine); + if (IS_ERR(rq)) + return PTR_ERR(rq); - map = i915_gem_object_pin_map(obj, I915_MAP_WC); - if (IS_ERR(map)) { - err = PTR_ERR(map); - goto err_obj; + fence = i915_active_fence_get(&ce->timeline->last_request); + if (fence) { + i915_request_await_dma_fence(rq, fence); + dma_fence_put(fence); } - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err_map; + rq = i915_request_get(rq); + i915_request_add(rq); + if (i915_request_wait(rq, 0, timeout) < 0) + err = -ETIME; + i915_request_put(rq); + + rmb(); /* We know the request is written, make sure all state is too! */ + return err; +} + +static int get_lri_mask(struct intel_engine_cs *engine, u32 lri) +{ + if ((lri & MI_LRI_LRM_CS_MMIO) == 0) + return ~0u; + + if (GRAPHICS_VER(engine->i915) < 12) + return 0xfff; + + switch (engine->class) { + default: + case RENDER_CLASS: + case COMPUTE_CLASS: + return 0x07ff; + case COPY_ENGINE_CLASS: + return 0x0fff; + case VIDEO_DECODE_CLASS: + case VIDEO_ENHANCEMENT_CLASS: + return 0x3fff; } +} - err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); - if (err) - goto err_map; +static int live_lrc_layout(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; + u32 *lrc; + int err; - for_each_engine(engine, i915, id) { - struct i915_request *lo, *hi; - struct igt_live_test t; - u32 *cs; + /* + * Check the registers offsets we use to create the initial reg state + * match the layout saved by HW. + */ + + lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */ + if (!lrc) + return -ENOMEM; + GEM_BUG_ON(offset_in_page(lrc)); + + err = 0; + for_each_engine(engine, gt, id) { + u32 *hw; + int dw; - if (!intel_engine_can_store_dword(engine)) + if (!engine->default_state) continue; - if (igt_live_test_begin(&t, i915, __func__, engine->name)) { - err = -EIO; - goto err_vma; + hw = shmem_pin_map(engine->default_state); + if (IS_ERR(hw)) { + err = PTR_ERR(hw); + break; } + hw += LRC_STATE_OFFSET / sizeof(*hw); - /* - * We create two requests. The low priority request - * busywaits on a semaphore (inside the ringbuffer where - * is should be preemptible) and the high priority requests - * uses a MI_STORE_DWORD_IMM to update the semaphore value - * allowing the first request to complete. If preemption - * fails, we hang instead. - */ - - lo = igt_request_alloc(ctx_lo, engine); - if (IS_ERR(lo)) { - err = PTR_ERR(lo); - goto err_vma; - } + __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE), + engine->kernel_context, engine, true); - cs = intel_ring_begin(lo, 8); - if (IS_ERR(cs)) { - err = PTR_ERR(cs); - i915_request_add(lo); - goto err_vma; - } + dw = 0; + do { + u32 lri = READ_ONCE(hw[dw]); + u32 lri_mask; - *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; - *cs++ = i915_ggtt_offset(vma); - *cs++ = 0; - *cs++ = 1; + if (lri == 0) { + dw++; + continue; + } - /* XXX Do we need a flush + invalidate here? */ + if (lrc[dw] == 0) { + pr_debug("%s: skipped instruction %x at dword %d\n", + engine->name, lri, dw); + dw++; + continue; + } - *cs++ = MI_SEMAPHORE_WAIT | - MI_SEMAPHORE_GLOBAL_GTT | - MI_SEMAPHORE_POLL | - MI_SEMAPHORE_SAD_EQ_SDD; - *cs++ = 0; - *cs++ = i915_ggtt_offset(vma); - *cs++ = 0; + if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + pr_err("%s: Expected LRI command at dword %d, found %08x\n", + engine->name, dw, lri); + err = -EINVAL; + break; + } - intel_ring_advance(lo, cs); - i915_request_add(lo); + if (lrc[dw] != lri) { + pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n", + engine->name, dw, lri, lrc[dw]); + err = -EINVAL; + break; + } - if (wait_for(READ_ONCE(*map), 10)) { - err = -ETIMEDOUT; - goto err_vma; - } + /* + * When bit 19 of MI_LOAD_REGISTER_IMM instruction + * opcode is set on Gen12+ devices, HW does not + * care about certain register address offsets, and + * instead check the following for valid address + * ranges on specific engines: + * RCS && CCS: BITS(0 - 10) + * BCS: BITS(0 - 11) + * VECS && VCS: BITS(0 - 13) + */ + lri_mask = get_lri_mask(engine, lri); + + lri &= 0x7f; + lri++; + dw++; + + while (lri) { + u32 offset = READ_ONCE(hw[dw]); + + if ((offset ^ lrc[dw]) & lri_mask) { + pr_err("%s: Different registers found at dword %d, expected %x, found %x\n", + engine->name, dw, offset, lrc[dw]); + err = -EINVAL; + break; + } - /* Low priority request should be busywaiting now */ - if (i915_request_wait(lo, 0, 1) != -ETIME) { - pr_err("%s: Busywaiting request did not!\n", - engine->name); - err = -EIO; - goto err_vma; - } + /* + * Skip over the actual register value as we + * expect that to differ. + */ + dw += 2; + lri -= 2; + } + } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); - hi = igt_request_alloc(ctx_hi, engine); - if (IS_ERR(hi)) { - err = PTR_ERR(hi); - goto err_vma; - } + if (err) { + pr_info("%s: HW register image:\n", engine->name); + igt_hexdump(hw, PAGE_SIZE); - cs = intel_ring_begin(hi, 4); - if (IS_ERR(cs)) { - err = PTR_ERR(cs); - i915_request_add(hi); - goto err_vma; + pr_info("%s: SW register image:\n", engine->name); + igt_hexdump(lrc, PAGE_SIZE); } - *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; - *cs++ = i915_ggtt_offset(vma); - *cs++ = 0; - *cs++ = 0; + shmem_unpin_map(engine->default_state, hw); + if (err) + break; + } - intel_ring_advance(hi, cs); - i915_request_add(hi); + free_page((unsigned long)lrc); + return err; +} - if (i915_request_wait(lo, 0, HZ / 5) < 0) { - struct drm_printer p = drm_info_printer(i915->drm.dev); +static int find_offset(const u32 *lri, u32 offset) +{ + int i; - pr_err("%s: Failed to preempt semaphore busywait!\n", - engine->name); + for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) + if (lri[i] == offset) + return i; - intel_engine_dump(engine, &p, "%s\n", engine->name); - GEM_TRACE_DUMP(); + return -1; +} - i915_gem_set_wedged(i915); - err = -EIO; - goto err_vma; +static int live_lrc_fixed(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; + int err = 0; + + /* + * Check the assumed register offsets match the actual locations in + * the context image. + */ + + for_each_engine(engine, gt, id) { + const struct { + u32 reg; + u32 offset; + const char *name; + } tbl[] = { + { + i915_mmio_reg_offset(RING_START(engine->mmio_base)), + CTX_RING_START - 1, + "RING_START" + }, + { + i915_mmio_reg_offset(RING_CTL(engine->mmio_base)), + CTX_RING_CTL - 1, + "RING_CTL" + }, + { + i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)), + CTX_RING_HEAD - 1, + "RING_HEAD" + }, + { + i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)), + CTX_RING_TAIL - 1, + "RING_TAIL" + }, + { + i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)), + lrc_ring_mi_mode(engine), + "RING_MI_MODE" + }, + { + i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)), + CTX_BB_STATE - 1, + "BB_STATE" + }, + { + i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)), + lrc_ring_wa_bb_per_ctx(engine), + "RING_BB_PER_CTX_PTR" + }, + { + i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)), + lrc_ring_indirect_ptr(engine), + "RING_INDIRECT_CTX_PTR" + }, + { + i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)), + lrc_ring_indirect_offset(engine), + "RING_INDIRECT_CTX_OFFSET" + }, + { + i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)), + CTX_TIMESTAMP - 1, + "RING_CTX_TIMESTAMP" + }, + { + i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)), + lrc_ring_gpr0(engine), + "RING_CS_GPR0" + }, + { + i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)), + lrc_ring_cmd_buf_cctl(engine), + "RING_CMD_BUF_CCTL" + }, + { }, + }, *t; + u32 *hw; + + if (!engine->default_state) + continue; + + hw = shmem_pin_map(engine->default_state); + if (IS_ERR(hw)) { + err = PTR_ERR(hw); + break; } - GEM_BUG_ON(READ_ONCE(*map)); + hw += LRC_STATE_OFFSET / sizeof(*hw); - if (igt_live_test_end(&t)) { - err = -EIO; - goto err_vma; + for (t = tbl; t->name; t++) { + int dw = find_offset(hw, t->reg); + + if (dw != t->offset) { + pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n", + engine->name, + t->name, + t->reg, + dw, + t->offset); + err = -EINVAL; + } } + + shmem_unpin_map(engine->default_state, hw); } - err = 0; -err_vma: - i915_vma_unpin(vma); -err_map: - i915_gem_object_unpin_map(obj); -err_obj: - i915_gem_object_put(obj); -err_ctx_lo: - kernel_context_close(ctx_lo); -err_ctx_hi: - kernel_context_close(ctx_hi); -err_unlock: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - err = -EIO; - intel_runtime_pm_put(&i915->runtime_pm, wakeref); - mutex_unlock(&i915->drm.struct_mutex); return err; } -static int live_preempt(void *arg) +static int __live_lrc_state(struct intel_engine_cs *engine, + struct i915_vma *scratch) { - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx_hi, *ctx_lo; - struct igt_spinner spin_hi, spin_lo; - struct intel_engine_cs *engine; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; + struct intel_context *ce; + struct i915_request *rq; + struct i915_gem_ww_ctx ww; + enum { + RING_START_IDX = 0, + RING_TAIL_IDX, + MAX_IDX + }; + u32 expected[MAX_IDX]; + u32 *cs; + int err; + int n; - if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) - pr_err("Logical preemption supported, but not exposed\n"); + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(&i915->runtime_pm); + i915_gem_ww_ctx_init(&ww, false); +retry: + err = i915_gem_object_lock(scratch->obj, &ww); + if (!err) + err = intel_context_pin_ww(ce, &ww); + if (err) + goto err_put; - if (igt_spinner_init(&spin_hi, i915)) - goto err_unlock; + rq = i915_request_create(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_unpin; + } - if (igt_spinner_init(&spin_lo, i915)) - goto err_spin_hi; + cs = intel_ring_begin(rq, 4 * MAX_IDX); + if (IS_ERR(cs)) { + err = PTR_ERR(cs); + i915_request_add(rq); + goto err_unpin; + } - ctx_hi = kernel_context(i915); - if (!ctx_hi) - goto err_spin_lo; - ctx_hi->sched.priority = - I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); + *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; + *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base)); + *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32); + *cs++ = 0; - ctx_lo = kernel_context(i915); - if (!ctx_lo) - goto err_ctx_hi; - ctx_lo->sched.priority = - I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); + expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma); - for_each_engine(engine, i915, id) { - struct igt_live_test t; - struct i915_request *rq; + *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; + *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)); + *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32); + *cs++ = 0; - if (!intel_engine_has_preemption(engine)) - continue; + err = i915_request_await_object(rq, scratch->obj, true); + if (!err) + err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); - if (igt_live_test_begin(&t, i915, __func__, engine->name)) { - err = -EIO; - goto err_ctx_lo; - } + i915_request_get(rq); + i915_request_add(rq); + if (err) + goto err_rq; - rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_ctx_lo; - } + intel_engine_flush_submission(engine); + expected[RING_TAIL_IDX] = ce->ring->tail; - i915_request_add(rq); - if (!igt_wait_for_spinner(&spin_lo, rq)) { - GEM_TRACE("lo spinner failed to start\n"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; - } + if (i915_request_wait(rq, 0, HZ / 5) < 0) { + err = -ETIME; + goto err_rq; + } - rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) { - igt_spinner_end(&spin_lo); - err = PTR_ERR(rq); - goto err_ctx_lo; - } + cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); + if (IS_ERR(cs)) { + err = PTR_ERR(cs); + goto err_rq; + } - i915_request_add(rq); - if (!igt_wait_for_spinner(&spin_hi, rq)) { - GEM_TRACE("hi spinner failed to start\n"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; + for (n = 0; n < MAX_IDX; n++) { + if (cs[n] != expected[n]) { + pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n", + engine->name, n, cs[n], expected[n]); + err = -EINVAL; + break; } + } - igt_spinner_end(&spin_hi); - igt_spinner_end(&spin_lo); + i915_gem_object_unpin_map(scratch->obj); - if (igt_live_test_end(&t)) { - err = -EIO; - goto err_ctx_lo; - } +err_rq: + i915_request_put(rq); +err_unpin: + intel_context_unpin(ce); +err_put: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) + goto retry; } - - err = 0; -err_ctx_lo: - kernel_context_close(ctx_lo); -err_ctx_hi: - kernel_context_close(ctx_hi); -err_spin_lo: - igt_spinner_fini(&spin_lo); -err_spin_hi: - igt_spinner_fini(&spin_hi); -err_unlock: - igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(&i915->runtime_pm, wakeref); - mutex_unlock(&i915->drm.struct_mutex); + i915_gem_ww_ctx_fini(&ww); + intel_context_put(ce); return err; } -static int live_late_preempt(void *arg) +static int live_lrc_state(void *arg) { - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx_hi, *ctx_lo; - struct igt_spinner spin_hi, spin_lo; + struct intel_gt *gt = arg; struct intel_engine_cs *engine; - struct i915_sched_attr attr = {}; + struct i915_vma *scratch; enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; + int err = 0; - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; + /* + * Check the live register state matches what we expect for this + * intel_context. + */ - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(&i915->runtime_pm); + scratch = create_scratch(gt); + if (IS_ERR(scratch)) + return PTR_ERR(scratch); - if (igt_spinner_init(&spin_hi, i915)) - goto err_unlock; + for_each_engine(engine, gt, id) { + err = __live_lrc_state(engine, scratch); + if (err) + break; + } - if (igt_spinner_init(&spin_lo, i915)) - goto err_spin_hi; + if (igt_flush_test(gt->i915)) + err = -EIO; - ctx_hi = kernel_context(i915); - if (!ctx_hi) - goto err_spin_lo; + i915_vma_unpin_and_release(&scratch, 0); + return err; +} - ctx_lo = kernel_context(i915); - if (!ctx_lo) - goto err_ctx_hi; +static int gpr_make_dirty(struct intel_context *ce) +{ + struct i915_request *rq; + u32 *cs; + int n; - for_each_engine(engine, i915, id) { - struct igt_live_test t; - struct i915_request *rq; + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return PTR_ERR(rq); - if (!intel_engine_has_preemption(engine)) - continue; + cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2); + if (IS_ERR(cs)) { + i915_request_add(rq); + return PTR_ERR(cs); + } - if (igt_live_test_begin(&t, i915, __func__, engine->name)) { - err = -EIO; - goto err_ctx_lo; - } + *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW); + for (n = 0; n < NUM_GPR_DW; n++) { + *cs++ = CS_GPR(ce->engine, n); + *cs++ = STACK_MAGIC; + } + *cs++ = MI_NOOP; - rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_ctx_lo; - } + intel_ring_advance(rq, cs); - i915_request_add(rq); - if (!igt_wait_for_spinner(&spin_lo, rq)) { - pr_err("First context failed to start\n"); - goto err_wedged; - } + rq->sched.attr.priority = I915_PRIORITY_BARRIER; + i915_request_add(rq); - rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, - MI_NOOP); - if (IS_ERR(rq)) { - igt_spinner_end(&spin_lo); - err = PTR_ERR(rq); - goto err_ctx_lo; - } + return 0; +} +static struct i915_request * +__gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot) +{ + const u32 offset = + i915_ggtt_offset(ce->engine->status_page.vma) + + offset_in_page(slot); + struct i915_request *rq; + u32 *cs; + int err; + int n; + + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return rq; + + cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW); + if (IS_ERR(cs)) { i915_request_add(rq); - if (igt_wait_for_spinner(&spin_hi, rq)) { - pr_err("Second context overtook first?\n"); - goto err_wedged; - } + return ERR_CAST(cs); + } - attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX); - engine->schedule(rq, &attr); + *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; + *cs++ = MI_NOOP; + + *cs++ = MI_SEMAPHORE_WAIT | + MI_SEMAPHORE_GLOBAL_GTT | + MI_SEMAPHORE_POLL | + MI_SEMAPHORE_SAD_NEQ_SDD; + *cs++ = 0; + *cs++ = offset; + *cs++ = 0; + + for (n = 0; n < NUM_GPR_DW; n++) { + *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; + *cs++ = CS_GPR(ce->engine, n); + *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32); + *cs++ = 0; + } - if (!igt_wait_for_spinner(&spin_hi, rq)) { - pr_err("High priority context failed to preempt the low priority context\n"); - GEM_TRACE_DUMP(); - goto err_wedged; - } + i915_vma_lock(scratch); + err = i915_request_await_object(rq, scratch->obj, true); + if (!err) + err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); + i915_vma_unlock(scratch); - igt_spinner_end(&spin_hi); - igt_spinner_end(&spin_lo); + i915_request_get(rq); + i915_request_add(rq); + if (err) { + i915_request_put(rq); + rq = ERR_PTR(err); + } - if (igt_live_test_end(&t)) { - err = -EIO; - goto err_ctx_lo; - } + return rq; +} + +static int __live_lrc_gpr(struct intel_engine_cs *engine, + struct i915_vma *scratch, + bool preempt) +{ + u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4); + struct intel_context *ce; + struct i915_request *rq; + u32 *cs; + int err; + int n; + + if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS) + return 0; /* GPR only on rcs0 for gen8 */ + + err = gpr_make_dirty(engine->kernel_context); + if (err) + return err; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + rq = __gpr_read(ce, scratch, slot); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_put; } - err = 0; -err_ctx_lo: - kernel_context_close(ctx_lo); -err_ctx_hi: - kernel_context_close(ctx_hi); -err_spin_lo: - igt_spinner_fini(&spin_lo); -err_spin_hi: - igt_spinner_fini(&spin_hi); -err_unlock: - igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(&i915->runtime_pm, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; + err = wait_for_submit(engine, rq, HZ / 2); + if (err) + goto err_rq; -err_wedged: - igt_spinner_end(&spin_hi); - igt_spinner_end(&spin_lo); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; -} + if (preempt) { + err = gpr_make_dirty(engine->kernel_context); + if (err) + goto err_rq; -struct preempt_client { - struct igt_spinner spin; - struct i915_gem_context *ctx; -}; + err = emit_semaphore_signal(engine->kernel_context, slot); + if (err) + goto err_rq; -static int preempt_client_init(struct drm_i915_private *i915, - struct preempt_client *c) -{ - c->ctx = kernel_context(i915); - if (!c->ctx) - return -ENOMEM; + err = wait_for_submit(engine, rq, HZ / 2); + if (err) + goto err_rq; + } else { + slot[0] = 1; + wmb(); + } - if (igt_spinner_init(&c->spin, i915)) - goto err_ctx; + if (i915_request_wait(rq, 0, HZ / 5) < 0) { + err = -ETIME; + goto err_rq; + } - return 0; + cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB); + if (IS_ERR(cs)) { + err = PTR_ERR(cs); + goto err_rq; + } -err_ctx: - kernel_context_close(c->ctx); - return -ENOMEM; -} + for (n = 0; n < NUM_GPR_DW; n++) { + if (cs[n]) { + pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n", + engine->name, + n / 2, n & 1 ? "udw" : "ldw", + cs[n]); + err = -EINVAL; + break; + } + } -static void preempt_client_fini(struct preempt_client *c) -{ - igt_spinner_fini(&c->spin); - kernel_context_close(c->ctx); + i915_gem_object_unpin_map(scratch->obj); + +err_rq: + memset32(&slot[0], -1, 4); + wmb(); + i915_request_put(rq); +err_put: + intel_context_put(ce); + return err; } -static int live_suppress_self_preempt(void *arg) +static int live_lrc_gpr(void *arg) { - struct drm_i915_private *i915 = arg; + struct intel_gt *gt = arg; struct intel_engine_cs *engine; - struct i915_sched_attr attr = { - .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX) - }; - struct preempt_client a, b; + struct i915_vma *scratch; enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; + int err = 0; /* - * Verify that if a preemption request does not cause a change in - * the current execution order, the preempt-to-idle injection is - * skipped and that we do not accidentally apply it after the CS - * completion event. + * Check that GPR registers are cleared in new contexts as we need + * to avoid leaking any information from previous contexts. */ - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; - - if (USES_GUC_SUBMISSION(i915)) - return 0; /* presume black blox */ + scratch = create_scratch(gt); + if (IS_ERR(scratch)) + return PTR_ERR(scratch); - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(&i915->runtime_pm); + for_each_engine(engine, gt, id) { + st_engine_heartbeat_disable(engine); - if (preempt_client_init(i915, &a)) - goto err_unlock; - if (preempt_client_init(i915, &b)) - goto err_client_a; - - for_each_engine(engine, i915, id) { - struct i915_request *rq_a, *rq_b; - int depth; + err = __live_lrc_gpr(engine, scratch, false); + if (err) + goto err; - if (!intel_engine_has_preemption(engine)) - continue; + err = __live_lrc_gpr(engine, scratch, true); + if (err) + goto err; - engine->execlists.preempt_hang.count = 0; +err: + st_engine_heartbeat_enable(engine); + if (igt_flush_test(gt->i915)) + err = -EIO; + if (err) + break; + } - rq_a = igt_spinner_create_request(&a.spin, - a.ctx, engine, - MI_NOOP); - if (IS_ERR(rq_a)) { - err = PTR_ERR(rq_a); - goto err_client_b; - } + i915_vma_unpin_and_release(&scratch, 0); + return err; +} - i915_request_add(rq_a); - if (!igt_wait_for_spinner(&a.spin, rq_a)) { - pr_err("First client failed to start\n"); - goto err_wedged; - } +static struct i915_request * +create_timestamp(struct intel_context *ce, void *slot, int idx) +{ + const u32 offset = + i915_ggtt_offset(ce->engine->status_page.vma) + + offset_in_page(slot); + struct i915_request *rq; + u32 *cs; + int err; - for (depth = 0; depth < 8; depth++) { - rq_b = igt_spinner_create_request(&b.spin, - b.ctx, engine, - MI_NOOP); - if (IS_ERR(rq_b)) { - err = PTR_ERR(rq_b); - goto err_client_b; - } - i915_request_add(rq_b); + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return rq; - GEM_BUG_ON(i915_request_completed(rq_a)); - engine->schedule(rq_a, &attr); - igt_spinner_end(&a.spin); + cs = intel_ring_begin(rq, 10); + if (IS_ERR(cs)) { + err = PTR_ERR(cs); + goto err; + } - if (!igt_wait_for_spinner(&b.spin, rq_b)) { - pr_err("Second client failed to start\n"); - goto err_wedged; - } + *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; + *cs++ = MI_NOOP; - swap(a, b); - rq_a = rq_b; - } - igt_spinner_end(&a.spin); + *cs++ = MI_SEMAPHORE_WAIT | + MI_SEMAPHORE_GLOBAL_GTT | + MI_SEMAPHORE_POLL | + MI_SEMAPHORE_SAD_NEQ_SDD; + *cs++ = 0; + *cs++ = offset; + *cs++ = 0; - if (engine->execlists.preempt_hang.count) { - pr_err("Preemption recorded x%d, depth %d; should have been suppressed!\n", - engine->execlists.preempt_hang.count, - depth); - err = -EINVAL; - goto err_client_b; - } + *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; + *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base)); + *cs++ = offset + idx * sizeof(u32); + *cs++ = 0; - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - goto err_wedged; - } + intel_ring_advance(rq, cs); err = 0; -err_client_b: - preempt_client_fini(&b); -err_client_a: - preempt_client_fini(&a); -err_unlock: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - err = -EIO; - intel_runtime_pm_put(&i915->runtime_pm, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; +err: + i915_request_get(rq); + i915_request_add(rq); + if (err) { + i915_request_put(rq); + return ERR_PTR(err); + } -err_wedged: - igt_spinner_end(&b.spin); - igt_spinner_end(&a.spin); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_client_b; + return rq; } -static int __i915_sw_fence_call -dummy_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) +struct lrc_timestamp { + struct intel_engine_cs *engine; + struct intel_context *ce[2]; + u32 poison; +}; + +static bool timestamp_advanced(u32 start, u32 end) { - return NOTIFY_DONE; + return (s32)(end - start) > 0; } -static struct i915_request *dummy_request(struct intel_engine_cs *engine) +static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt) { + u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4); struct i915_request *rq; + u32 timestamp; + int err = 0; - rq = kzalloc(sizeof(*rq), GFP_KERNEL); - if (!rq) - return NULL; - - INIT_LIST_HEAD(&rq->active_list); - rq->engine = engine; - - i915_sched_node_init(&rq->sched); - - /* mark this request as permanently incomplete */ - rq->fence.seqno = 1; - BUILD_BUG_ON(sizeof(rq->fence.seqno) != 8); /* upper 32b == 0 */ - rq->hwsp_seqno = (u32 *)&rq->fence.seqno + 1; - GEM_BUG_ON(i915_request_completed(rq)); + arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison; + rq = create_timestamp(arg->ce[0], slot, 1); + if (IS_ERR(rq)) + return PTR_ERR(rq); - i915_sw_fence_init(&rq->submit, dummy_notify); - set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); + err = wait_for_submit(rq->engine, rq, HZ / 2); + if (err) + goto err; - return rq; -} + if (preempt) { + arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef; + err = emit_semaphore_signal(arg->ce[1], slot); + if (err) + goto err; + } else { + slot[0] = 1; + wmb(); + } -static void dummy_request_free(struct i915_request *dummy) -{ - /* We have to fake the CS interrupt to kick the next request */ - i915_sw_fence_commit(&dummy->submit); + /* And wait for switch to kernel (to save our context to memory) */ + err = context_flush(arg->ce[0], HZ / 2); + if (err) + goto err; - i915_request_mark_complete(dummy); - dma_fence_signal(&dummy->fence); + if (!timestamp_advanced(arg->poison, slot[1])) { + pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n", + arg->engine->name, preempt ? "preempt" : "simple", + arg->poison, slot[1]); + err = -EINVAL; + } - i915_sched_node_fini(&dummy->sched); - i915_sw_fence_fini(&dummy->submit); + timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]); + if (!timestamp_advanced(slot[1], timestamp)) { + pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n", + arg->engine->name, preempt ? "preempt" : "simple", + slot[1], timestamp); + err = -EINVAL; + } - dma_fence_free(&dummy->fence); +err: + memset32(slot, -1, 4); + i915_request_put(rq); + return err; } -static int live_suppress_wait_preempt(void *arg) +static int live_lrc_timestamp(void *arg) { - struct drm_i915_private *i915 = arg; - struct preempt_client client[4]; - struct intel_engine_cs *engine; + struct lrc_timestamp data = {}; + struct intel_gt *gt = arg; enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - int i; + const u32 poison[] = { + 0, + S32_MAX, + (u32)S32_MAX + 1, + U32_MAX, + }; /* - * Waiters are given a little priority nudge, but not enough - * to actually cause any preemption. Double check that we do - * not needlessly generate preempt-to-idle cycles. + * We want to verify that the timestamp is saved and restore across + * context switches and is monotonic. + * + * So we do this with a little bit of LRC poisoning to check various + * boundary conditions, and see what happens if we preempt the context + * with a second request (carrying more poison into the timestamp). */ - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; + for_each_engine(data.engine, gt, id) { + int i, err = 0; - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(&i915->runtime_pm); + st_engine_heartbeat_disable(data.engine); - if (preempt_client_init(i915, &client[0])) /* ELSP[0] */ - goto err_unlock; - if (preempt_client_init(i915, &client[1])) /* ELSP[1] */ - goto err_client_0; - if (preempt_client_init(i915, &client[2])) /* head of queue */ - goto err_client_1; - if (preempt_client_init(i915, &client[3])) /* bystander */ - goto err_client_2; + for (i = 0; i < ARRAY_SIZE(data.ce); i++) { + struct intel_context *tmp; - for_each_engine(engine, i915, id) { - int depth; + tmp = intel_context_create(data.engine); + if (IS_ERR(tmp)) { + err = PTR_ERR(tmp); + goto err; + } - if (!intel_engine_has_preemption(engine)) - continue; + err = intel_context_pin(tmp); + if (err) { + intel_context_put(tmp); + goto err; + } - if (!engine->emit_init_breadcrumb) - continue; + data.ce[i] = tmp; + } - for (depth = 0; depth < ARRAY_SIZE(client); depth++) { - struct i915_request *rq[ARRAY_SIZE(client)]; - struct i915_request *dummy; + for (i = 0; i < ARRAY_SIZE(poison); i++) { + data.poison = poison[i]; - engine->execlists.preempt_hang.count = 0; + err = __lrc_timestamp(&data, false); + if (err) + break; - dummy = dummy_request(engine); - if (!dummy) - goto err_client_3; + err = __lrc_timestamp(&data, true); + if (err) + break; + } - for (i = 0; i < ARRAY_SIZE(client); i++) { - rq[i] = igt_spinner_create_request(&client[i].spin, - client[i].ctx, engine, - MI_NOOP); - if (IS_ERR(rq[i])) { - err = PTR_ERR(rq[i]); - goto err_wedged; - } +err: + st_engine_heartbeat_enable(data.engine); + for (i = 0; i < ARRAY_SIZE(data.ce); i++) { + if (!data.ce[i]) + break; - /* Disable NEWCLIENT promotion */ - __i915_active_request_set(&rq[i]->timeline->last_request, - dummy); - i915_request_add(rq[i]); - } + intel_context_unpin(data.ce[i]); + intel_context_put(data.ce[i]); + } - dummy_request_free(dummy); + if (igt_flush_test(gt->i915)) + err = -EIO; + if (err) + return err; + } - GEM_BUG_ON(i915_request_completed(rq[0])); - if (!igt_wait_for_spinner(&client[0].spin, rq[0])) { - pr_err("%s: First client failed to start\n", - engine->name); - goto err_wedged; - } - GEM_BUG_ON(!i915_request_started(rq[0])); - - if (i915_request_wait(rq[depth], - I915_WAIT_PRIORITY, - 1) != -ETIME) { - pr_err("%s: Waiter depth:%d completed!\n", - engine->name, depth); - goto err_wedged; - } + return 0; +} - for (i = 0; i < ARRAY_SIZE(client); i++) - igt_spinner_end(&client[i].spin); +static struct i915_vma * +create_user_vma(struct i915_address_space *vm, unsigned long size) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + int err; - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - goto err_wedged; + obj = i915_gem_object_create_internal(vm->i915, size); + if (IS_ERR(obj)) + return ERR_CAST(obj); - if (engine->execlists.preempt_hang.count) { - pr_err("%s: Preemption recorded x%d, depth %d; should have been suppressed!\n", - engine->name, - engine->execlists.preempt_hang.count, - depth); - err = -EINVAL; - goto err_client_3; - } - } + vma = i915_vma_instance(obj, vm, NULL); + if (IS_ERR(vma)) { + i915_gem_object_put(obj); + return vma; } - err = 0; -err_client_3: - preempt_client_fini(&client[3]); -err_client_2: - preempt_client_fini(&client[2]); -err_client_1: - preempt_client_fini(&client[1]); -err_client_0: - preempt_client_fini(&client[0]); -err_unlock: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - err = -EIO; - intel_runtime_pm_put(&i915->runtime_pm, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; + err = i915_vma_pin(vma, 0, 0, PIN_USER); + if (err) { + i915_gem_object_put(obj); + return ERR_PTR(err); + } -err_wedged: - for (i = 0; i < ARRAY_SIZE(client); i++) - igt_spinner_end(&client[i].spin); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_client_3; + return vma; } -static int live_chain_preempt(void *arg) +static u32 safe_poison(u32 offset, u32 poison) { - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct preempt_client hi, lo; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - /* - * Build a chain AB...BA between two contexts (A, B) and request - * preemption of the last request. It should then complete before - * the previously submitted spinner in B. + * Do not enable predication as it will nop all subsequent commands, + * not only disabling the tests (by preventing all the other SRM) but + * also preventing the arbitration events at the end of the request. */ + if (offset == i915_mmio_reg_offset(RING_PREDICATE_RESULT(0))) + poison &= ~REG_BIT(0); - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; + return poison; +} + +static struct i915_vma * +store_context(struct intel_context *ce, struct i915_vma *scratch) +{ + struct i915_vma *batch; + u32 dw, x, *cs, *hw; + u32 *defaults; - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(&i915->runtime_pm); + batch = create_user_vma(ce->vm, SZ_64K); + if (IS_ERR(batch)) + return batch; + + cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); + if (IS_ERR(cs)) { + i915_vma_put(batch); + return ERR_CAST(cs); + } - if (preempt_client_init(i915, &hi)) - goto err_unlock; + defaults = shmem_pin_map(ce->engine->default_state); + if (!defaults) { + i915_gem_object_unpin_map(batch->obj); + i915_vma_put(batch); + return ERR_PTR(-ENOMEM); + } - if (preempt_client_init(i915, &lo)) - goto err_client_hi; + x = 0; + dw = 0; + hw = defaults; + hw += LRC_STATE_OFFSET / sizeof(*hw); + do { + u32 len = hw[dw] & 0x7f; - for_each_engine(engine, i915, id) { - struct i915_sched_attr attr = { - .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX), - }; - struct igt_live_test t; - struct i915_request *rq; - int ring_size, count, i; + if (hw[dw] == 0) { + dw++; + continue; + } - if (!intel_engine_has_preemption(engine)) + if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + dw += len + 2; continue; + } - rq = igt_spinner_create_request(&lo.spin, - lo.ctx, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); + dw++; + len = (len + 1) / 2; + while (len--) { + *cs++ = MI_STORE_REGISTER_MEM_GEN8; + *cs++ = hw[dw]; + *cs++ = lower_32_bits(scratch->node.start + x); + *cs++ = upper_32_bits(scratch->node.start + x); - ring_size = rq->wa_tail - rq->head; - if (ring_size < 0) - ring_size += rq->ring->size; - ring_size = rq->ring->size / ring_size; - pr_debug("%s(%s): Using maximum of %d requests\n", - __func__, engine->name, ring_size); - - igt_spinner_end(&lo.spin); - if (i915_request_wait(rq, 0, HZ / 2) < 0) { - pr_err("Timed out waiting to flush %s\n", engine->name); - goto err_wedged; + dw += 2; + x += 4; } + } while (dw < PAGE_SIZE / sizeof(u32) && + (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); - if (igt_live_test_begin(&t, i915, __func__, engine->name)) { - err = -EIO; - goto err_wedged; - } + *cs++ = MI_BATCH_BUFFER_END; - for_each_prime_number_from(count, 1, ring_size) { - rq = igt_spinner_create_request(&hi.spin, - hi.ctx, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); - if (!igt_wait_for_spinner(&hi.spin, rq)) - goto err_wedged; - - rq = igt_spinner_create_request(&lo.spin, - lo.ctx, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); + shmem_unpin_map(ce->engine->default_state, defaults); - for (i = 0; i < count; i++) { - rq = igt_request_alloc(lo.ctx, engine); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); - } + i915_gem_object_flush_map(batch->obj); + i915_gem_object_unpin_map(batch->obj); - rq = igt_request_alloc(hi.ctx, engine); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); - engine->schedule(rq, &attr); - - igt_spinner_end(&hi.spin); - if (i915_request_wait(rq, 0, HZ / 5) < 0) { - struct drm_printer p = - drm_info_printer(i915->drm.dev); - - pr_err("Failed to preempt over chain of %d\n", - count); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - goto err_wedged; - } - igt_spinner_end(&lo.spin); + return batch; +} - rq = igt_request_alloc(lo.ctx, engine); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); - if (i915_request_wait(rq, 0, HZ / 5) < 0) { - struct drm_printer p = - drm_info_printer(i915->drm.dev); - - pr_err("Failed to flush low priority chain of %d requests\n", - count); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - goto err_wedged; - } - } +static int move_to_active(struct i915_request *rq, + struct i915_vma *vma, + unsigned int flags) +{ + int err; - if (igt_live_test_end(&t)) { - err = -EIO; - goto err_wedged; - } - } + i915_vma_lock(vma); + err = i915_request_await_object(rq, vma->obj, flags); + if (!err) + err = i915_vma_move_to_active(vma, rq, flags); + i915_vma_unlock(vma); - err = 0; -err_client_lo: - preempt_client_fini(&lo); -err_client_hi: - preempt_client_fini(&hi); -err_unlock: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - err = -EIO; - intel_runtime_pm_put(&i915->runtime_pm, wakeref); - mutex_unlock(&i915->drm.struct_mutex); return err; - -err_wedged: - igt_spinner_end(&hi.spin); - igt_spinner_end(&lo.spin); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_client_lo; } -static int live_preempt_hang(void *arg) +static struct i915_request * +record_registers(struct intel_context *ce, + struct i915_vma *before, + struct i915_vma *after, + u32 *sema) { - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx_hi, *ctx_lo; - struct igt_spinner spin_hi, spin_lo; - struct intel_engine_cs *engine; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; + struct i915_vma *b_before, *b_after; + struct i915_request *rq; + u32 *cs; + int err; - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; + b_before = store_context(ce, before); + if (IS_ERR(b_before)) + return ERR_CAST(b_before); - if (!intel_has_reset_engine(i915)) - return 0; + b_after = store_context(ce, after); + if (IS_ERR(b_after)) { + rq = ERR_CAST(b_after); + goto err_before; + } - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(&i915->runtime_pm); + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + goto err_after; - if (igt_spinner_init(&spin_hi, i915)) - goto err_unlock; + err = move_to_active(rq, before, EXEC_OBJECT_WRITE); + if (err) + goto err_rq; - if (igt_spinner_init(&spin_lo, i915)) - goto err_spin_hi; + err = move_to_active(rq, b_before, 0); + if (err) + goto err_rq; - ctx_hi = kernel_context(i915); - if (!ctx_hi) - goto err_spin_lo; - ctx_hi->sched.priority = - I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); + err = move_to_active(rq, after, EXEC_OBJECT_WRITE); + if (err) + goto err_rq; - ctx_lo = kernel_context(i915); - if (!ctx_lo) - goto err_ctx_hi; - ctx_lo->sched.priority = - I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); + err = move_to_active(rq, b_after, 0); + if (err) + goto err_rq; - for_each_engine(engine, i915, id) { - struct i915_request *rq; + cs = intel_ring_begin(rq, 14); + if (IS_ERR(cs)) { + err = PTR_ERR(cs); + goto err_rq; + } - if (!intel_engine_has_preemption(engine)) - continue; + *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; + *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); + *cs++ = lower_32_bits(b_before->node.start); + *cs++ = upper_32_bits(b_before->node.start); + + *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; + *cs++ = MI_SEMAPHORE_WAIT | + MI_SEMAPHORE_GLOBAL_GTT | + MI_SEMAPHORE_POLL | + MI_SEMAPHORE_SAD_NEQ_SDD; + *cs++ = 0; + *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + + offset_in_page(sema); + *cs++ = 0; + *cs++ = MI_NOOP; + + *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; + *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); + *cs++ = lower_32_bits(b_after->node.start); + *cs++ = upper_32_bits(b_after->node.start); + + intel_ring_advance(rq, cs); + + WRITE_ONCE(*sema, 0); + i915_request_get(rq); + i915_request_add(rq); +err_after: + i915_vma_put(b_after); +err_before: + i915_vma_put(b_before); + return rq; + +err_rq: + i915_request_add(rq); + rq = ERR_PTR(err); + goto err_after; +} + +static struct i915_vma *load_context(struct intel_context *ce, u32 poison) +{ + struct i915_vma *batch; + u32 dw, *cs, *hw; + u32 *defaults; - rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_ctx_lo; + batch = create_user_vma(ce->vm, SZ_64K); + if (IS_ERR(batch)) + return batch; + + cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); + if (IS_ERR(cs)) { + i915_vma_put(batch); + return ERR_CAST(cs); + } + + defaults = shmem_pin_map(ce->engine->default_state); + if (!defaults) { + i915_gem_object_unpin_map(batch->obj); + i915_vma_put(batch); + return ERR_PTR(-ENOMEM); + } + + dw = 0; + hw = defaults; + hw += LRC_STATE_OFFSET / sizeof(*hw); + do { + u32 len = hw[dw] & 0x7f; + + if (hw[dw] == 0) { + dw++; + continue; } - i915_request_add(rq); - if (!igt_wait_for_spinner(&spin_lo, rq)) { - GEM_TRACE("lo spinner failed to start\n"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; + if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + dw += len + 2; + continue; } - rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) { - igt_spinner_end(&spin_lo); - err = PTR_ERR(rq); - goto err_ctx_lo; + dw++; + len = (len + 1) / 2; + *cs++ = MI_LOAD_REGISTER_IMM(len); + while (len--) { + *cs++ = hw[dw]; + *cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine, + MI_LRI_LRM_CS_MMIO), + poison); + dw += 2; } + } while (dw < PAGE_SIZE / sizeof(u32) && + (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); - init_completion(&engine->execlists.preempt_hang.completion); - engine->execlists.preempt_hang.inject_hang = true; + *cs++ = MI_BATCH_BUFFER_END; - i915_request_add(rq); + shmem_unpin_map(ce->engine->default_state, defaults); - if (!wait_for_completion_timeout(&engine->execlists.preempt_hang.completion, - HZ / 10)) { - pr_err("Preemption did not occur within timeout!"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; - } + i915_gem_object_flush_map(batch->obj); + i915_gem_object_unpin_map(batch->obj); - set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); - i915_reset_engine(engine, NULL); - clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); + return batch; +} - engine->execlists.preempt_hang.inject_hang = false; +static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema) +{ + struct i915_request *rq; + struct i915_vma *batch; + u32 *cs; + int err; - if (!igt_wait_for_spinner(&spin_hi, rq)) { - GEM_TRACE("hi spinner failed to start\n"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; - } + batch = load_context(ce, poison); + if (IS_ERR(batch)) + return PTR_ERR(batch); - igt_spinner_end(&spin_hi); - igt_spinner_end(&spin_lo); - if (igt_flush_test(i915, I915_WAIT_LOCKED)) { - err = -EIO; - goto err_ctx_lo; - } + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_batch; } - err = 0; -err_ctx_lo: - kernel_context_close(ctx_lo); -err_ctx_hi: - kernel_context_close(ctx_hi); -err_spin_lo: - igt_spinner_fini(&spin_lo); -err_spin_hi: - igt_spinner_fini(&spin_hi); -err_unlock: - igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(&i915->runtime_pm, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; -} + err = move_to_active(rq, batch, 0); + if (err) + goto err_rq; -static int random_range(struct rnd_state *rnd, int min, int max) -{ - return i915_prandom_u32_max_state(max - min, rnd) + min; -} + cs = intel_ring_begin(rq, 8); + if (IS_ERR(cs)) { + err = PTR_ERR(cs); + goto err_rq; + } -static int random_priority(struct rnd_state *rnd) -{ - return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX); -} + *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; + *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); + *cs++ = lower_32_bits(batch->node.start); + *cs++ = upper_32_bits(batch->node.start); -struct preempt_smoke { - struct drm_i915_private *i915; - struct i915_gem_context **contexts; - struct intel_engine_cs *engine; - struct drm_i915_gem_object *batch; - unsigned int ncontext; - struct rnd_state prng; - unsigned long count; -}; + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + + offset_in_page(sema); + *cs++ = 0; + *cs++ = 1; + + intel_ring_advance(rq, cs); -static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke) + rq->sched.attr.priority = I915_PRIORITY_BARRIER; +err_rq: + i915_request_add(rq); +err_batch: + i915_vma_put(batch); + return err; +} + +static bool is_moving(u32 a, u32 b) { - return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext, - &smoke->prng)]; + return a != b; } -static int smoke_submit(struct preempt_smoke *smoke, - struct i915_gem_context *ctx, int prio, - struct drm_i915_gem_object *batch) +static int compare_isolation(struct intel_engine_cs *engine, + struct i915_vma *ref[2], + struct i915_vma *result[2], + struct intel_context *ce, + u32 poison) { - struct i915_request *rq; - struct i915_vma *vma = NULL; + u32 x, dw, *hw, *lrc; + u32 *A[2], *B[2]; + u32 *defaults; int err = 0; - if (batch) { - vma = i915_vma_instance(batch, ctx->vm, NULL); - if (IS_ERR(vma)) - return PTR_ERR(vma); + A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC); + if (IS_ERR(A[0])) + return PTR_ERR(A[0]); - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - return err; + A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC); + if (IS_ERR(A[1])) { + err = PTR_ERR(A[1]); + goto err_A0; } - ctx->sched.priority = prio; + B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC); + if (IS_ERR(B[0])) { + err = PTR_ERR(B[0]); + goto err_A1; + } - rq = igt_request_alloc(ctx, smoke->engine); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto unpin; + B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC); + if (IS_ERR(B[1])) { + err = PTR_ERR(B[1]); + goto err_B0; } - if (vma) { - i915_vma_lock(vma); - err = rq->engine->emit_bb_start(rq, - vma->node.start, - PAGE_SIZE, 0); - if (!err) - err = i915_vma_move_to_active(vma, rq, 0); - i915_vma_unlock(vma); + lrc = i915_gem_object_pin_map_unlocked(ce->state->obj, + i915_coherent_map_type(engine->i915, + ce->state->obj, + false)); + if (IS_ERR(lrc)) { + err = PTR_ERR(lrc); + goto err_B1; } + lrc += LRC_STATE_OFFSET / sizeof(*hw); - i915_request_add(rq); + defaults = shmem_pin_map(ce->engine->default_state); + if (!defaults) { + err = -ENOMEM; + goto err_lrc; + } + + x = 0; + dw = 0; + hw = defaults; + hw += LRC_STATE_OFFSET / sizeof(*hw); + do { + u32 len = hw[dw] & 0x7f; -unpin: - if (vma) - i915_vma_unpin(vma); + if (hw[dw] == 0) { + dw++; + continue; + } + + if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + dw += len + 2; + continue; + } + dw++; + len = (len + 1) / 2; + while (len--) { + if (!is_moving(A[0][x], A[1][x]) && + (A[0][x] != B[0][x] || A[1][x] != B[1][x])) { + switch (hw[dw] & 4095) { + case 0x30: /* RING_HEAD */ + case 0x34: /* RING_TAIL */ + break; + + default: + pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n", + engine->name, dw, + hw[dw], hw[dw + 1], + A[0][x], B[0][x], B[1][x], + poison, lrc[dw + 1]); + err = -EINVAL; + } + } + dw += 2; + x++; + } + } while (dw < PAGE_SIZE / sizeof(u32) && + (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); + + shmem_unpin_map(ce->engine->default_state, defaults); +err_lrc: + i915_gem_object_unpin_map(ce->state->obj); +err_B1: + i915_gem_object_unpin_map(result[1]->obj); +err_B0: + i915_gem_object_unpin_map(result[0]->obj); +err_A1: + i915_gem_object_unpin_map(ref[1]->obj); +err_A0: + i915_gem_object_unpin_map(ref[0]->obj); return err; } -static int smoke_crescendo_thread(void *arg) +static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) { - struct preempt_smoke *smoke = arg; - IGT_TIMEOUT(end_time); - unsigned long count; + u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1); + struct i915_vma *ref[2], *result[2]; + struct intel_context *A, *B; + struct i915_request *rq; + int err; - count = 0; - do { - struct i915_gem_context *ctx = smoke_context(smoke); - int err; - - mutex_lock(&smoke->i915->drm.struct_mutex); - err = smoke_submit(smoke, - ctx, count % I915_PRIORITY_MAX, - smoke->batch); - mutex_unlock(&smoke->i915->drm.struct_mutex); - if (err) - return err; + A = intel_context_create(engine); + if (IS_ERR(A)) + return PTR_ERR(A); - count++; - } while (!__igt_timeout(end_time, NULL)); + B = intel_context_create(engine); + if (IS_ERR(B)) { + err = PTR_ERR(B); + goto err_A; + } - smoke->count = count; - return 0; -} + ref[0] = create_user_vma(A->vm, SZ_64K); + if (IS_ERR(ref[0])) { + err = PTR_ERR(ref[0]); + goto err_B; + } -static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags) -#define BATCH BIT(0) -{ - struct task_struct *tsk[I915_NUM_ENGINES] = {}; - struct preempt_smoke arg[I915_NUM_ENGINES]; - struct intel_engine_cs *engine; - enum intel_engine_id id; - unsigned long count; - int err = 0; + ref[1] = create_user_vma(A->vm, SZ_64K); + if (IS_ERR(ref[1])) { + err = PTR_ERR(ref[1]); + goto err_ref0; + } - mutex_unlock(&smoke->i915->drm.struct_mutex); + rq = record_registers(A, ref[0], ref[1], sema); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_ref1; + } - for_each_engine(engine, smoke->i915, id) { - arg[id] = *smoke; - arg[id].engine = engine; - if (!(flags & BATCH)) - arg[id].batch = NULL; - arg[id].count = 0; + WRITE_ONCE(*sema, 1); + wmb(); - tsk[id] = kthread_run(smoke_crescendo_thread, &arg, - "igt/smoke:%d", id); - if (IS_ERR(tsk[id])) { - err = PTR_ERR(tsk[id]); - break; - } - get_task_struct(tsk[id]); + if (i915_request_wait(rq, 0, HZ / 2) < 0) { + i915_request_put(rq); + err = -ETIME; + goto err_ref1; } + i915_request_put(rq); - count = 0; - for_each_engine(engine, smoke->i915, id) { - int status; + result[0] = create_user_vma(A->vm, SZ_64K); + if (IS_ERR(result[0])) { + err = PTR_ERR(result[0]); + goto err_ref1; + } - if (IS_ERR_OR_NULL(tsk[id])) - continue; + result[1] = create_user_vma(A->vm, SZ_64K); + if (IS_ERR(result[1])) { + err = PTR_ERR(result[1]); + goto err_result0; + } - status = kthread_stop(tsk[id]); - if (status && !err) - err = status; + rq = record_registers(A, result[0], result[1], sema); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_result1; + } - count += arg[id].count; + err = poison_registers(B, poison, sema); + if (err) { + WRITE_ONCE(*sema, -1); + i915_request_put(rq); + goto err_result1; + } - put_task_struct(tsk[id]); + if (i915_request_wait(rq, 0, HZ / 2) < 0) { + i915_request_put(rq); + err = -ETIME; + goto err_result1; } + i915_request_put(rq); + + err = compare_isolation(engine, ref, result, A, poison); + +err_result1: + i915_vma_put(result[1]); +err_result0: + i915_vma_put(result[0]); +err_ref1: + i915_vma_put(ref[1]); +err_ref0: + i915_vma_put(ref[0]); +err_B: + intel_context_put(B); +err_A: + intel_context_put(A); + return err; +} - mutex_lock(&smoke->i915->drm.struct_mutex); +static bool skip_isolation(const struct intel_engine_cs *engine) +{ + if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9) + return true; - pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n", - count, flags, - RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext); - return 0; + if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11) + return true; + + return false; } -static int smoke_random(struct preempt_smoke *smoke, unsigned int flags) +static int live_lrc_isolation(void *arg) { + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; enum intel_engine_id id; - IGT_TIMEOUT(end_time); - unsigned long count; + const u32 poison[] = { + STACK_MAGIC, + 0x3a3a3a3a, + 0x5c5c5c5c, + 0xffffffff, + 0xffff0000, + }; + int err = 0; - count = 0; - do { - for_each_engine(smoke->engine, smoke->i915, id) { - struct i915_gem_context *ctx = smoke_context(smoke); - int err; + /* + * Our goal is try and verify that per-context state cannot be + * tampered with by another non-privileged client. + * + * We take the list of context registers from the LRI in the default + * context image and attempt to modify that list from a remote context. + */ - err = smoke_submit(smoke, - ctx, random_priority(&smoke->prng), - flags & BATCH ? smoke->batch : NULL); - if (err) - return err; + for_each_engine(engine, gt, id) { + int i; - count++; + /* Just don't even ask */ + if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) && + skip_isolation(engine)) + continue; + + intel_engine_pm_get(engine); + for (i = 0; i < ARRAY_SIZE(poison); i++) { + int result; + + result = __lrc_isolation(engine, poison[i]); + if (result && !err) + err = result; + + result = __lrc_isolation(engine, ~poison[i]); + if (result && !err) + err = result; } - } while (!__igt_timeout(end_time, NULL)); + intel_engine_pm_put(engine); + if (igt_flush_test(gt->i915)) { + err = -EIO; + break; + } + } - pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n", - count, flags, - RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext); - return 0; + return err; } -static int live_preempt_smoke(void *arg) +static int indirect_ctx_submit_req(struct intel_context *ce) { - struct preempt_smoke smoke = { - .i915 = arg, - .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed), - .ncontext = 1024, - }; - const unsigned int phase[] = { 0, BATCH }; - intel_wakeref_t wakeref; - struct igt_live_test t; - int err = -ENOMEM; - u32 *cs; - int n; + struct i915_request *rq; + int err = 0; - if (!HAS_LOGICAL_RING_PREEMPTION(smoke.i915)) - return 0; + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return PTR_ERR(rq); - smoke.contexts = kmalloc_array(smoke.ncontext, - sizeof(*smoke.contexts), - GFP_KERNEL); - if (!smoke.contexts) - return -ENOMEM; + i915_request_get(rq); + i915_request_add(rq); - mutex_lock(&smoke.i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(&smoke.i915->runtime_pm); + if (i915_request_wait(rq, 0, HZ / 5) < 0) + err = -ETIME; - smoke.batch = i915_gem_object_create_internal(smoke.i915, PAGE_SIZE); - if (IS_ERR(smoke.batch)) { - err = PTR_ERR(smoke.batch); - goto err_unlock; - } + i915_request_put(rq); - cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB); - if (IS_ERR(cs)) { - err = PTR_ERR(cs); - goto err_batch; - } - for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++) - cs[n] = MI_ARB_CHECK; - cs[n] = MI_BATCH_BUFFER_END; - i915_gem_object_flush_map(smoke.batch); - i915_gem_object_unpin_map(smoke.batch); + return err; +} - if (igt_live_test_begin(&t, smoke.i915, __func__, "all")) { - err = -EIO; - goto err_batch; - } +#define CTX_BB_CANARY_OFFSET (3 * 1024) +#define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32)) - for (n = 0; n < smoke.ncontext; n++) { - smoke.contexts[n] = kernel_context(smoke.i915); - if (!smoke.contexts[n]) - goto err_ctx; - } +static u32 * +emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs) +{ + *cs++ = MI_STORE_REGISTER_MEM_GEN8 | + MI_SRM_LRM_GLOBAL_GTT | + MI_LRI_LRM_CS_MMIO; + *cs++ = i915_mmio_reg_offset(RING_START(0)); + *cs++ = i915_ggtt_offset(ce->state) + + context_wa_bb_offset(ce) + + CTX_BB_CANARY_OFFSET; + *cs++ = 0; + + return cs; +} - for (n = 0; n < ARRAY_SIZE(phase); n++) { - err = smoke_crescendo(&smoke, phase[n]); - if (err) - goto err_ctx; +static void +indirect_ctx_bb_setup(struct intel_context *ce) +{ + u32 *cs = context_indirect_bb(ce); - err = smoke_random(&smoke, phase[n]); - if (err) - goto err_ctx; - } + cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d; -err_ctx: - if (igt_live_test_end(&t)) - err = -EIO; + setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary); +} - for (n = 0; n < smoke.ncontext; n++) { - if (!smoke.contexts[n]) - break; - kernel_context_close(smoke.contexts[n]); - } +static bool check_ring_start(struct intel_context *ce) +{ + const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) - + LRC_STATE_OFFSET + context_wa_bb_offset(ce); -err_batch: - i915_gem_object_put(smoke.batch); -err_unlock: - intel_runtime_pm_put(&smoke.i915->runtime_pm, wakeref); - mutex_unlock(&smoke.i915->drm.struct_mutex); - kfree(smoke.contexts); + if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START]) + return true; - return err; + pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n", + ctx_bb[CTX_BB_CANARY_INDEX], + ce->lrc_reg_state[CTX_RING_START]); + + return false; } -static int nop_virtual_engine(struct drm_i915_private *i915, - struct intel_engine_cs **siblings, - unsigned int nsibling, - unsigned int nctx, - unsigned int flags) -#define CHAIN BIT(0) +static int indirect_ctx_bb_check(struct intel_context *ce) { - IGT_TIMEOUT(end_time); - struct i915_request *request[16]; - struct i915_gem_context *ctx[16]; - struct intel_context *ve[16]; - unsigned long n, prime, nc; - struct igt_live_test t; - ktime_t times[2] = {}; int err; - GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ctx)); + err = indirect_ctx_submit_req(ce); + if (err) + return err; - for (n = 0; n < nctx; n++) { - ctx[n] = kernel_context(i915); - if (!ctx[n]) { - err = -ENOMEM; - nctx = n; - goto out; - } + if (!check_ring_start(ce)) + return -EINVAL; - ve[n] = intel_execlists_create_virtual(ctx[n], - siblings, nsibling); - if (IS_ERR(ve[n])) { - kernel_context_close(ctx[n]); - err = PTR_ERR(ve[n]); - nctx = n; - goto out; - } + return 0; +} - err = intel_context_pin(ve[n]); - if (err) { - intel_context_put(ve[n]); - kernel_context_close(ctx[n]); - nctx = n; - goto out; - } - } +static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine) +{ + struct intel_context *a, *b; + int err; - err = igt_live_test_begin(&t, i915, __func__, ve[0]->engine->name); + a = intel_context_create(engine); + if (IS_ERR(a)) + return PTR_ERR(a); + err = intel_context_pin(a); if (err) - goto out; - - for_each_prime_number_from(prime, 1, 8192) { - times[1] = ktime_get_raw(); - - if (flags & CHAIN) { - for (nc = 0; nc < nctx; nc++) { - for (n = 0; n < prime; n++) { - request[nc] = - i915_request_create(ve[nc]); - if (IS_ERR(request[nc])) { - err = PTR_ERR(request[nc]); - goto out; - } - - i915_request_add(request[nc]); - } - } - } else { - for (n = 0; n < prime; n++) { - for (nc = 0; nc < nctx; nc++) { - request[nc] = - i915_request_create(ve[nc]); - if (IS_ERR(request[nc])) { - err = PTR_ERR(request[nc]); - goto out; - } - - i915_request_add(request[nc]); - } - } - } - - for (nc = 0; nc < nctx; nc++) { - if (i915_request_wait(request[nc], 0, HZ / 10) < 0) { - pr_err("%s(%s): wait for %llx:%lld timed out\n", - __func__, ve[0]->engine->name, - request[nc]->fence.context, - request[nc]->fence.seqno); - - GEM_TRACE("%s(%s) failed at request %llx:%lld\n", - __func__, ve[0]->engine->name, - request[nc]->fence.context, - request[nc]->fence.seqno); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - break; - } - } + goto put_a; - times[1] = ktime_sub(ktime_get_raw(), times[1]); - if (prime == 1) - times[0] = times[1]; + b = intel_context_create(engine); + if (IS_ERR(b)) { + err = PTR_ERR(b); + goto unpin_a; + } + err = intel_context_pin(b); + if (err) + goto put_b; - if (__igt_timeout(end_time, NULL)) - break; + /* We use the already reserved extra page in context state */ + if (!a->wa_bb_page) { + GEM_BUG_ON(b->wa_bb_page); + GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12); + goto unpin_b; } - err = igt_live_test_end(&t); + /* + * In order to test that our per context bb is truly per context, + * and executes at the intended spot on context restoring process, + * make the batch store the ring start value to memory. + * As ring start is restored apriori of starting the indirect ctx bb and + * as it will be different for each context, it fits to this purpose. + */ + indirect_ctx_bb_setup(a); + indirect_ctx_bb_setup(b); + + err = indirect_ctx_bb_check(a); if (err) - goto out; + goto unpin_b; - pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n", - nctx, ve[0]->engine->name, ktime_to_ns(times[0]), - prime, div64_u64(ktime_to_ns(times[1]), prime)); + err = indirect_ctx_bb_check(b); -out: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - err = -EIO; +unpin_b: + intel_context_unpin(b); +put_b: + intel_context_put(b); +unpin_a: + intel_context_unpin(a); +put_a: + intel_context_put(a); - for (nc = 0; nc < nctx; nc++) { - intel_context_unpin(ve[nc]); - intel_context_put(ve[nc]); - kernel_context_close(ctx[nc]); - } return err; } -static int live_virtual_engine(void *arg) +static int live_lrc_indirect_ctx_bb(void *arg) { - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; + struct intel_gt *gt = arg; struct intel_engine_cs *engine; enum intel_engine_id id; - unsigned int class, inst; - int err = -ENODEV; + int err = 0; - if (USES_GUC_SUBMISSION(i915)) - return 0; + for_each_engine(engine, gt, id) { + intel_engine_pm_get(engine); + err = __live_lrc_indirect_ctx_bb(engine); + intel_engine_pm_put(engine); - mutex_lock(&i915->drm.struct_mutex); + if (igt_flush_test(gt->i915)) + err = -EIO; - for_each_engine(engine, i915, id) { - err = nop_virtual_engine(i915, &engine, 1, 1, 0); - if (err) { - pr_err("Failed to wrap engine %s: err=%d\n", - engine->name, err); - goto out_unlock; - } + if (err) + break; } - for (class = 0; class <= MAX_ENGINE_CLASS; class++) { - int nsibling, n; + return err; +} - nsibling = 0; - for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { - if (!i915->engine_class[class][inst]) - continue; +static void garbage_reset(struct intel_engine_cs *engine, + struct i915_request *rq) +{ + const unsigned int bit = I915_RESET_ENGINE + engine->id; + unsigned long *lock = &engine->gt->reset.flags; - siblings[nsibling++] = i915->engine_class[class][inst]; - } - if (nsibling < 2) - continue; + local_bh_disable(); + if (!test_and_set_bit(bit, lock)) { + tasklet_disable(&engine->sched_engine->tasklet); - for (n = 1; n <= nsibling + 1; n++) { - err = nop_virtual_engine(i915, siblings, nsibling, - n, 0); - if (err) - goto out_unlock; - } + if (!rq->fence.error) + __intel_engine_reset_bh(engine, NULL); - err = nop_virtual_engine(i915, siblings, nsibling, n, CHAIN); - if (err) - goto out_unlock; + tasklet_enable(&engine->sched_engine->tasklet); + clear_and_wake_up_bit(bit, lock); } - -out_unlock: - mutex_unlock(&i915->drm.struct_mutex); - return err; + local_bh_enable(); } -static int mask_virtual_engine(struct drm_i915_private *i915, - struct intel_engine_cs **siblings, - unsigned int nsibling) +static struct i915_request *garbage(struct intel_context *ce, + struct rnd_state *prng) { - struct i915_request *request[MAX_ENGINE_INSTANCE + 1]; - struct i915_gem_context *ctx; - struct intel_context *ve; - struct igt_live_test t; - unsigned int n; + struct i915_request *rq; int err; - /* - * Check that by setting the execution mask on a request, we can - * restrict it to our desired engine within the virtual engine. - */ + err = intel_context_pin(ce); + if (err) + return ERR_PTR(err); - ctx = kernel_context(i915); - if (!ctx) - return -ENOMEM; + prandom_bytes_state(prng, + ce->lrc_reg_state, + ce->engine->context_size - + LRC_STATE_OFFSET); - ve = intel_execlists_create_virtual(ctx, siblings, nsibling); - if (IS_ERR(ve)) { - err = PTR_ERR(ve); - goto out_close; + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_unpin; } - err = intel_context_pin(ve); - if (err) - goto out_put; + i915_request_get(rq); + i915_request_add(rq); + return rq; - err = igt_live_test_begin(&t, i915, __func__, ve->engine->name); - if (err) - goto out_unpin; - - for (n = 0; n < nsibling; n++) { - request[n] = i915_request_create(ve); - if (IS_ERR(request[n])) { - err = PTR_ERR(request[n]); - nsibling = n; - goto out; - } +err_unpin: + intel_context_unpin(ce); + return ERR_PTR(err); +} + +static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng) +{ + struct intel_context *ce; + struct i915_request *hang; + int err = 0; - /* Reverse order as it's more likely to be unnatural */ - request[n]->execution_mask = siblings[nsibling - n - 1]->mask; + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); - i915_request_get(request[n]); - i915_request_add(request[n]); + hang = garbage(ce, prng); + if (IS_ERR(hang)) { + err = PTR_ERR(hang); + goto err_ce; } - for (n = 0; n < nsibling; n++) { - if (i915_request_wait(request[n], 0, HZ / 10) < 0) { - pr_err("%s(%s): wait for %llx:%lld timed out\n", - __func__, ve->engine->name, - request[n]->fence.context, - request[n]->fence.seqno); + if (wait_for_submit(engine, hang, HZ / 2)) { + i915_request_put(hang); + err = -ETIME; + goto err_ce; + } - GEM_TRACE("%s(%s) failed at request %llx:%lld\n", - __func__, ve->engine->name, - request[n]->fence.context, - request[n]->fence.seqno); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto out; - } + intel_context_set_banned(ce); + garbage_reset(engine, hang); - if (request[n]->engine != siblings[nsibling - n - 1]) { - pr_err("Executed on wrong sibling '%s', expected '%s'\n", - request[n]->engine->name, - siblings[nsibling - n - 1]->name); - err = -EINVAL; - goto out; - } + intel_engine_flush_submission(engine); + if (!hang->fence.error) { + i915_request_put(hang); + pr_err("%s: corrupted context was not reset\n", + engine->name); + err = -EINVAL; + goto err_ce; } - err = igt_live_test_end(&t); - if (err) - goto out; - -out: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) + if (i915_request_wait(hang, 0, HZ / 2) < 0) { + pr_err("%s: corrupted context did not recover\n", + engine->name); + i915_request_put(hang); err = -EIO; + goto err_ce; + } + i915_request_put(hang); - for (n = 0; n < nsibling; n++) - i915_request_put(request[n]); - -out_unpin: - intel_context_unpin(ve); -out_put: - intel_context_put(ve); -out_close: - kernel_context_close(ctx); +err_ce: + intel_context_put(ce); return err; } -static int live_virtual_mask(void *arg) +static int live_lrc_garbage(void *arg) { - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; - unsigned int class, inst; - int err = 0; + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; + + /* + * Verify that we can recover if one context state is completely + * corrupted. + */ - if (USES_GUC_SUBMISSION(i915)) + if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN)) return 0; - mutex_lock(&i915->drm.struct_mutex); + for_each_engine(engine, gt, id) { + I915_RND_STATE(prng); + int err = 0, i; - for (class = 0; class <= MAX_ENGINE_CLASS; class++) { - unsigned int nsibling; + if (!intel_has_reset_engine(engine->gt)) + continue; - nsibling = 0; - for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { - if (!i915->engine_class[class][inst]) + intel_engine_pm_get(engine); + for (i = 0; i < 3; i++) { + err = __lrc_garbage(engine, &prng); + if (err) break; - - siblings[nsibling++] = i915->engine_class[class][inst]; } - if (nsibling < 2) - continue; + intel_engine_pm_put(engine); - err = mask_virtual_engine(i915, siblings, nsibling); + if (igt_flush_test(gt->i915)) + err = -EIO; if (err) - goto out_unlock; + return err; } -out_unlock: - mutex_unlock(&i915->drm.struct_mutex); - return err; + return 0; } -static int bond_virtual_engine(struct drm_i915_private *i915, - unsigned int class, - struct intel_engine_cs **siblings, - unsigned int nsibling, - unsigned int flags) -#define BOND_SCHEDULE BIT(0) +static int __live_pphwsp_runtime(struct intel_engine_cs *engine) { - struct intel_engine_cs *master; - struct i915_gem_context *ctx; - struct i915_request *rq[16]; - enum intel_engine_id id; - unsigned long n; + struct intel_context *ce; + struct i915_request *rq; + IGT_TIMEOUT(end_time); int err; - GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1); + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); - ctx = kernel_context(i915); - if (!ctx) - return -ENOMEM; - - err = 0; - rq[0] = ERR_PTR(-ENOMEM); - for_each_engine(master, i915, id) { - struct i915_sw_fence fence = {}; + ce->stats.runtime.num_underflow = 0; + ce->stats.runtime.max_underflow = 0; - if (master->class == class) - continue; + do { + unsigned int loop = 1024; - memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq)); + while (loop) { + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_rq; + } - rq[0] = igt_request_alloc(ctx, master); - if (IS_ERR(rq[0])) { - err = PTR_ERR(rq[0]); - goto out; - } - i915_request_get(rq[0]); + if (--loop == 0) + i915_request_get(rq); - if (flags & BOND_SCHEDULE) { - onstack_fence_init(&fence); - err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit, - &fence, - GFP_KERNEL); + i915_request_add(rq); } - i915_request_add(rq[0]); - if (err < 0) - goto out; - - for (n = 0; n < nsibling; n++) { - struct intel_context *ve; - - ve = intel_execlists_create_virtual(ctx, - siblings, - nsibling); - if (IS_ERR(ve)) { - err = PTR_ERR(ve); - onstack_fence_fini(&fence); - goto out; - } - - err = intel_virtual_engine_attach_bond(ve->engine, - master, - siblings[n]); - if (err) { - intel_context_put(ve); - onstack_fence_fini(&fence); - goto out; - } - err = intel_context_pin(ve); - intel_context_put(ve); - if (err) { - onstack_fence_fini(&fence); - goto out; - } + if (__igt_timeout(end_time, NULL)) + break; - rq[n + 1] = i915_request_create(ve); - intel_context_unpin(ve); - if (IS_ERR(rq[n + 1])) { - err = PTR_ERR(rq[n + 1]); - onstack_fence_fini(&fence); - goto out; - } - i915_request_get(rq[n + 1]); - - err = i915_request_await_execution(rq[n + 1], - &rq[0]->fence, - ve->engine->bond_execute); - i915_request_add(rq[n + 1]); - if (err < 0) { - onstack_fence_fini(&fence); - goto out; - } - } - onstack_fence_fini(&fence); + i915_request_put(rq); + } while (1); - if (i915_request_wait(rq[0], 0, HZ / 10) < 0) { - pr_err("Master request did not execute (on %s)!\n", - rq[0]->engine->name); - err = -EIO; - goto out; - } + err = i915_request_wait(rq, 0, HZ / 5); + if (err < 0) { + pr_err("%s: request not completed!\n", engine->name); + goto err_wait; + } - for (n = 0; n < nsibling; n++) { - if (i915_request_wait(rq[n + 1], 0, - MAX_SCHEDULE_TIMEOUT) < 0) { - err = -EIO; - goto out; - } + igt_flush_test(engine->i915); - if (rq[n + 1]->engine != siblings[n]) { - pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n", - siblings[n]->name, - rq[n + 1]->engine->name, - rq[0]->engine->name); - err = -EINVAL; - goto out; - } - } + pr_info("%s: pphwsp runtime %lluns, average %lluns\n", + engine->name, + intel_context_get_total_runtime_ns(ce), + intel_context_get_avg_runtime_ns(ce)); - for (n = 0; !IS_ERR(rq[n]); n++) - i915_request_put(rq[n]); - rq[0] = ERR_PTR(-ENOMEM); + err = 0; + if (ce->stats.runtime.num_underflow) { + pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n", + engine->name, + ce->stats.runtime.num_underflow, + ce->stats.runtime.max_underflow); + GEM_TRACE_DUMP(); + err = -EOVERFLOW; } -out: - for (n = 0; !IS_ERR(rq[n]); n++) - i915_request_put(rq[n]); - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - err = -EIO; - - kernel_context_close(ctx); +err_wait: + i915_request_put(rq); +err_rq: + intel_context_put(ce); return err; } -static int live_virtual_bond(void *arg) +static int live_pphwsp_runtime(void *arg) { - static const struct phase { - const char *name; - unsigned int flags; - } phases[] = { - { "", 0 }, - { "schedule", BOND_SCHEDULE }, - { }, - }; - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; - unsigned int class, inst; + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; int err = 0; - if (USES_GUC_SUBMISSION(i915)) - return 0; - - mutex_lock(&i915->drm.struct_mutex); - - for (class = 0; class <= MAX_ENGINE_CLASS; class++) { - const struct phase *p; - int nsibling; - - nsibling = 0; - for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { - if (!i915->engine_class[class][inst]) - break; - - GEM_BUG_ON(nsibling == ARRAY_SIZE(siblings)); - siblings[nsibling++] = i915->engine_class[class][inst]; - } - if (nsibling < 2) - continue; + /* + * Check that cumulative context runtime as stored in the pphwsp[16] + * is monotonic. + */ - for (p = phases; p->name; p++) { - err = bond_virtual_engine(i915, - class, siblings, nsibling, - p->flags); - if (err) { - pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n", - __func__, p->name, class, nsibling, err); - goto out_unlock; - } - } + for_each_engine(engine, gt, id) { + err = __live_pphwsp_runtime(engine); + if (err) + break; } -out_unlock: - mutex_unlock(&i915->drm.struct_mutex); + if (igt_flush_test(gt->i915)) + err = -EIO; + return err; } -int intel_execlists_live_selftests(struct drm_i915_private *i915) +int intel_lrc_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { - SUBTEST(live_sanitycheck), - SUBTEST(live_busywait_preempt), - SUBTEST(live_preempt), - SUBTEST(live_late_preempt), - SUBTEST(live_suppress_self_preempt), - SUBTEST(live_suppress_wait_preempt), - SUBTEST(live_chain_preempt), - SUBTEST(live_preempt_hang), - SUBTEST(live_preempt_smoke), - SUBTEST(live_virtual_engine), - SUBTEST(live_virtual_mask), - SUBTEST(live_virtual_bond), + SUBTEST(live_lrc_layout), + SUBTEST(live_lrc_fixed), + SUBTEST(live_lrc_state), + SUBTEST(live_lrc_gpr), + SUBTEST(live_lrc_isolation), + SUBTEST(live_lrc_timestamp), + SUBTEST(live_lrc_garbage), + SUBTEST(live_pphwsp_runtime), + SUBTEST(live_lrc_indirect_ctx_bb), }; - if (!HAS_EXECLISTS(i915)) - return 0; - - if (i915_terminally_wedged(i915)) + if (!HAS_LOGICAL_RING_CONTEXTS(i915)) return 0; - return i915_subtests(tests, i915); + return intel_gt_live_subtests(tests, to_gt(i915)); } |