aboutsummaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/i915/intel_engine_cs.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/intel_engine_cs.c')
-rw-r--r--drivers/gpu/drm/i915/intel_engine_cs.c491
1 files changed, 277 insertions, 214 deletions
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 49fa43ff02ba..eea9bec04f1b 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -84,7 +84,6 @@ static const struct engine_class_info intel_engine_classes[] = {
#define MAX_MMIO_BASES 3
struct engine_info {
unsigned int hw_id;
- unsigned int uabi_id;
u8 class;
u8 instance;
/* mmio bases table *must* be sorted in reverse gen order */
@@ -95,27 +94,24 @@ struct engine_info {
};
static const struct engine_info intel_engines[] = {
- [RCS] = {
- .hw_id = RCS_HW,
- .uabi_id = I915_EXEC_RENDER,
+ [RCS0] = {
+ .hw_id = RCS0_HW,
.class = RENDER_CLASS,
.instance = 0,
.mmio_bases = {
{ .gen = 1, .base = RENDER_RING_BASE }
},
},
- [BCS] = {
- .hw_id = BCS_HW,
- .uabi_id = I915_EXEC_BLT,
+ [BCS0] = {
+ .hw_id = BCS0_HW,
.class = COPY_ENGINE_CLASS,
.instance = 0,
.mmio_bases = {
{ .gen = 6, .base = BLT_RING_BASE }
},
},
- [VCS] = {
- .hw_id = VCS_HW,
- .uabi_id = I915_EXEC_BSD,
+ [VCS0] = {
+ .hw_id = VCS0_HW,
.class = VIDEO_DECODE_CLASS,
.instance = 0,
.mmio_bases = {
@@ -124,9 +120,8 @@ static const struct engine_info intel_engines[] = {
{ .gen = 4, .base = BSD_RING_BASE }
},
},
- [VCS2] = {
- .hw_id = VCS2_HW,
- .uabi_id = I915_EXEC_BSD,
+ [VCS1] = {
+ .hw_id = VCS1_HW,
.class = VIDEO_DECODE_CLASS,
.instance = 1,
.mmio_bases = {
@@ -134,27 +129,24 @@ static const struct engine_info intel_engines[] = {
{ .gen = 8, .base = GEN8_BSD2_RING_BASE }
},
},
- [VCS3] = {
- .hw_id = VCS3_HW,
- .uabi_id = I915_EXEC_BSD,
+ [VCS2] = {
+ .hw_id = VCS2_HW,
.class = VIDEO_DECODE_CLASS,
.instance = 2,
.mmio_bases = {
{ .gen = 11, .base = GEN11_BSD3_RING_BASE }
},
},
- [VCS4] = {
- .hw_id = VCS4_HW,
- .uabi_id = I915_EXEC_BSD,
+ [VCS3] = {
+ .hw_id = VCS3_HW,
.class = VIDEO_DECODE_CLASS,
.instance = 3,
.mmio_bases = {
{ .gen = 11, .base = GEN11_BSD4_RING_BASE }
},
},
- [VECS] = {
- .hw_id = VECS_HW,
- .uabi_id = I915_EXEC_VEBOX,
+ [VECS0] = {
+ .hw_id = VECS0_HW,
.class = VIDEO_ENHANCEMENT_CLASS,
.instance = 0,
.mmio_bases = {
@@ -162,9 +154,8 @@ static const struct engine_info intel_engines[] = {
{ .gen = 7, .base = VEBOX_RING_BASE }
},
},
- [VECS2] = {
- .hw_id = VECS2_HW,
- .uabi_id = I915_EXEC_VEBOX,
+ [VECS1] = {
+ .hw_id = VECS1_HW,
.class = VIDEO_ENHANCEMENT_CLASS,
.instance = 1,
.mmio_bases = {
@@ -264,21 +255,17 @@ static void __sprint_engine_name(char *name, const struct engine_info *info)
void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
{
- struct drm_i915_private *dev_priv = engine->i915;
- i915_reg_t hwstam;
-
/*
* Though they added more rings on g4x/ilk, they did not add
* per-engine HWSTAM until gen6.
*/
- if (INTEL_GEN(dev_priv) < 6 && engine->class != RENDER_CLASS)
+ if (INTEL_GEN(engine->i915) < 6 && engine->class != RENDER_CLASS)
return;
- hwstam = RING_HWSTAM(engine->mmio_base);
- if (INTEL_GEN(dev_priv) >= 3)
- I915_WRITE(hwstam, mask);
+ if (INTEL_GEN(engine->i915) >= 3)
+ ENGINE_WRITE(engine, RING_HWSTAM, mask);
else
- I915_WRITE16(hwstam, mask);
+ ENGINE_WRITE16(engine, RING_HWSTAM, mask);
}
static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
@@ -313,15 +300,18 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
if (!engine)
return -ENOMEM;
+ BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES);
+
engine->id = id;
+ engine->mask = BIT(id);
engine->i915 = dev_priv;
+ engine->uncore = &dev_priv->uncore;
__sprint_engine_name(engine->name, info);
engine->hw_id = engine->guc_id = info->hw_id;
engine->mmio_base = __engine_mmio_base(dev_priv, info->mmio_bases);
engine->class = info->class;
engine->instance = info->instance;
- engine->uabi_id = info->uabi_id;
engine->uabi_class = intel_engine_classes[info->class].uabi_class;
engine->context_size = __intel_engine_context_size(dev_priv,
@@ -355,15 +345,15 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
{
struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
- const unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
+ const unsigned int engine_mask = INTEL_INFO(dev_priv)->engine_mask;
struct intel_engine_cs *engine;
enum intel_engine_id id;
unsigned int mask = 0;
unsigned int i;
int err;
- WARN_ON(ring_mask == 0);
- WARN_ON(ring_mask &
+ WARN_ON(engine_mask == 0);
+ WARN_ON(engine_mask &
GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
if (i915_inject_load_failure())
@@ -377,7 +367,7 @@ int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
if (err)
goto cleanup;
- mask |= ENGINE_MASK(i);
+ mask |= BIT(i);
}
/*
@@ -385,16 +375,16 @@ int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
* are added to the driver by a warning and disabling the forgotten
* engines.
*/
- if (WARN_ON(mask != ring_mask))
- device_info->ring_mask = mask;
+ if (WARN_ON(mask != engine_mask))
+ device_info->engine_mask = mask;
/* We always presume we have at least RCS available for later probing */
- if (WARN_ON(!HAS_ENGINE(dev_priv, RCS))) {
+ if (WARN_ON(!HAS_ENGINE(dev_priv, RCS0))) {
err = -ENODEV;
goto cleanup;
}
- RUNTIME_INFO(dev_priv)->num_rings = hweight32(mask);
+ RUNTIME_INFO(dev_priv)->num_engines = hweight32(mask);
i915_check_and_clear_faults(dev_priv);
@@ -455,12 +445,6 @@ cleanup:
return err;
}
-void intel_engine_write_global_seqno(struct intel_engine_cs *engine, u32 seqno)
-{
- intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
- GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno);
-}
-
static void intel_engine_init_batch_pool(struct intel_engine_cs *engine)
{
i915_gem_batch_pool_init(&engine->batch_pool, engine);
@@ -541,9 +525,7 @@ static int init_status_page(struct intel_engine_cs *engine)
return PTR_ERR(obj);
}
- ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
- if (ret)
- goto err;
+ i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
if (IS_ERR(vma)) {
@@ -594,7 +576,6 @@ int intel_engine_setup_common(struct intel_engine_cs *engine)
err = i915_timeline_init(engine->i915,
&engine->timeline,
- engine->name,
engine->status_page.vma);
if (err)
goto err_hwsp;
@@ -614,10 +595,44 @@ err_hwsp:
return err;
}
-static void __intel_context_unpin(struct i915_gem_context *ctx,
- struct intel_engine_cs *engine)
+void intel_engines_set_scheduler_caps(struct drm_i915_private *i915)
{
- intel_context_unpin(to_intel_context(ctx, engine));
+ static const struct {
+ u8 engine;
+ u8 sched;
+ } map[] = {
+#define MAP(x, y) { ilog2(I915_ENGINE_HAS_##x), ilog2(I915_SCHEDULER_CAP_##y) }
+ MAP(PREEMPTION, PREEMPTION),
+ MAP(SEMAPHORES, SEMAPHORES),
+#undef MAP
+ };
+ struct intel_engine_cs *engine;
+ enum intel_engine_id id;
+ u32 enabled, disabled;
+
+ enabled = 0;
+ disabled = 0;
+ for_each_engine(engine, i915, id) { /* all engines must agree! */
+ int i;
+
+ if (engine->schedule)
+ enabled |= (I915_SCHEDULER_CAP_ENABLED |
+ I915_SCHEDULER_CAP_PRIORITY);
+ else
+ disabled |= (I915_SCHEDULER_CAP_ENABLED |
+ I915_SCHEDULER_CAP_PRIORITY);
+
+ for (i = 0; i < ARRAY_SIZE(map); i++) {
+ if (engine->flags & BIT(map[i].engine))
+ enabled |= BIT(map[i].sched);
+ else
+ disabled |= BIT(map[i].sched);
+ }
+ }
+
+ i915->caps.scheduler = enabled & ~disabled;
+ if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_ENABLED))
+ i915->caps.scheduler = 0;
}
struct measure_breadcrumb {
@@ -639,7 +654,7 @@ static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
return -ENOMEM;
if (i915_timeline_init(engine->i915,
- &frame->timeline, "measure",
+ &frame->timeline,
engine->status_page.vma))
goto out_frame;
@@ -670,6 +685,20 @@ out_frame:
return dw;
}
+static int pin_context(struct i915_gem_context *ctx,
+ struct intel_engine_cs *engine,
+ struct intel_context **out)
+{
+ struct intel_context *ce;
+
+ ce = intel_context_pin(ctx, engine);
+ if (IS_ERR(ce))
+ return PTR_ERR(ce);
+
+ *out = ce;
+ return 0;
+}
+
/**
* intel_engines_init_common - initialize cengine state which might require hw access
* @engine: Engine to initialize.
@@ -684,11 +713,8 @@ out_frame:
int intel_engine_init_common(struct intel_engine_cs *engine)
{
struct drm_i915_private *i915 = engine->i915;
- struct intel_context *ce;
int ret;
- engine->set_default_submission(engine);
-
/* We may need to do things with the shrinker which
* require us to immediately switch back to the default
* context. This can cause a problem as pinning the
@@ -696,39 +722,61 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
* be available. To avoid this we always pin the default
* context.
*/
- ce = intel_context_pin(i915->kernel_context, engine);
- if (IS_ERR(ce))
- return PTR_ERR(ce);
+ ret = pin_context(i915->kernel_context, engine,
+ &engine->kernel_context);
+ if (ret)
+ return ret;
/*
* Similarly the preempt context must always be available so that
- * we can interrupt the engine at any time.
+ * we can interrupt the engine at any time. However, as preemption
+ * is optional, we allow it to fail.
*/
- if (i915->preempt_context) {
- ce = intel_context_pin(i915->preempt_context, engine);
- if (IS_ERR(ce)) {
- ret = PTR_ERR(ce);
- goto err_unpin_kernel;
- }
- }
+ if (i915->preempt_context)
+ pin_context(i915->preempt_context, engine,
+ &engine->preempt_context);
ret = measure_breadcrumb_dw(engine);
if (ret < 0)
- goto err_unpin_preempt;
+ goto err_unpin;
engine->emit_fini_breadcrumb_dw = ret;
- return 0;
+ engine->set_default_submission(engine);
-err_unpin_preempt:
- if (i915->preempt_context)
- __intel_context_unpin(i915->preempt_context, engine);
+ return 0;
-err_unpin_kernel:
- __intel_context_unpin(i915->kernel_context, engine);
+err_unpin:
+ if (engine->preempt_context)
+ intel_context_unpin(engine->preempt_context);
+ intel_context_unpin(engine->kernel_context);
return ret;
}
+void intel_gt_resume(struct drm_i915_private *i915)
+{
+ struct intel_engine_cs *engine;
+ enum intel_engine_id id;
+
+ /*
+ * After resume, we may need to poke into the pinned kernel
+ * contexts to paper over any damage caused by the sudden suspend.
+ * Only the kernel contexts should remain pinned over suspend,
+ * allowing us to fixup the user contexts on their first pin.
+ */
+ for_each_engine(engine, i915, id) {
+ struct intel_context *ce;
+
+ ce = engine->kernel_context;
+ if (ce)
+ ce->ops->reset(ce);
+
+ ce = engine->preempt_context;
+ if (ce)
+ ce->ops->reset(ce);
+ }
+}
+
/**
* intel_engines_cleanup_common - cleans up the engine state created by
* the common initiailizers.
@@ -738,8 +786,6 @@ err_unpin_kernel:
*/
void intel_engine_cleanup_common(struct intel_engine_cs *engine)
{
- struct drm_i915_private *i915 = engine->i915;
-
cleanup_status_page(engine);
intel_engine_fini_breadcrumbs(engine);
@@ -749,9 +795,9 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
if (engine->default_state)
i915_gem_object_put(engine->default_state);
- if (i915->preempt_context)
- __intel_context_unpin(i915->preempt_context, engine);
- __intel_context_unpin(i915->kernel_context, engine);
+ if (engine->preempt_context)
+ intel_context_unpin(engine->preempt_context);
+ intel_context_unpin(engine->kernel_context);
i915_timeline_fini(&engine->timeline);
@@ -762,50 +808,48 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
{
- struct drm_i915_private *dev_priv = engine->i915;
+ struct drm_i915_private *i915 = engine->i915;
+
u64 acthd;
- if (INTEL_GEN(dev_priv) >= 8)
- acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
- RING_ACTHD_UDW(engine->mmio_base));
- else if (INTEL_GEN(dev_priv) >= 4)
- acthd = I915_READ(RING_ACTHD(engine->mmio_base));
+ if (INTEL_GEN(i915) >= 8)
+ acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW);
+ else if (INTEL_GEN(i915) >= 4)
+ acthd = ENGINE_READ(engine, RING_ACTHD);
else
- acthd = I915_READ(ACTHD);
+ acthd = ENGINE_READ(engine, ACTHD);
return acthd;
}
u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
{
- struct drm_i915_private *dev_priv = engine->i915;
u64 bbaddr;
- if (INTEL_GEN(dev_priv) >= 8)
- bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
- RING_BBADDR_UDW(engine->mmio_base));
+ if (INTEL_GEN(engine->i915) >= 8)
+ bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW);
else
- bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
+ bbaddr = ENGINE_READ(engine, RING_BBADDR);
return bbaddr;
}
int intel_engine_stop_cs(struct intel_engine_cs *engine)
{
- struct drm_i915_private *dev_priv = engine->i915;
+ struct intel_uncore *uncore = engine->uncore;
const u32 base = engine->mmio_base;
const i915_reg_t mode = RING_MI_MODE(base);
int err;
- if (INTEL_GEN(dev_priv) < 3)
+ if (INTEL_GEN(engine->i915) < 3)
return -ENODEV;
GEM_TRACE("%s\n", engine->name);
- I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
+ intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING));
err = 0;
- if (__intel_wait_for_register_fw(dev_priv,
+ if (__intel_wait_for_register_fw(uncore,
mode, MODE_IDLE, MODE_IDLE,
1000, 0,
NULL)) {
@@ -814,19 +858,16 @@ int intel_engine_stop_cs(struct intel_engine_cs *engine)
}
/* A final mmio read to let GPU writes be hopefully flushed to memory */
- POSTING_READ_FW(mode);
+ intel_uncore_posting_read_fw(uncore, mode);
return err;
}
void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
{
- struct drm_i915_private *dev_priv = engine->i915;
-
GEM_TRACE("%s\n", engine->name);
- I915_WRITE_FW(RING_MI_MODE(engine->mmio_base),
- _MASKED_BIT_DISABLE(STOP_RING));
+ ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
}
const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
@@ -863,6 +904,7 @@ static inline u32
read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
int subslice, i915_reg_t reg)
{
+ struct intel_uncore *uncore = &dev_priv->uncore;
u32 mcr_slice_subslice_mask;
u32 mcr_slice_subslice_select;
u32 default_mcr_s_ss_select;
@@ -884,33 +926,33 @@ read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
default_mcr_s_ss_select = intel_calculate_mcr_s_ss_select(dev_priv);
- fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
+ fw_domains = intel_uncore_forcewake_for_reg(uncore, reg,
FW_REG_READ);
- fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
+ fw_domains |= intel_uncore_forcewake_for_reg(uncore,
GEN8_MCR_SELECTOR,
FW_REG_READ | FW_REG_WRITE);
- spin_lock_irq(&dev_priv->uncore.lock);
- intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
+ spin_lock_irq(&uncore->lock);
+ intel_uncore_forcewake_get__locked(uncore, fw_domains);
- mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
+ mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
WARN_ON_ONCE((mcr & mcr_slice_subslice_mask) !=
default_mcr_s_ss_select);
mcr &= ~mcr_slice_subslice_mask;
mcr |= mcr_slice_subslice_select;
- I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
+ intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
- ret = I915_READ_FW(reg);
+ ret = intel_uncore_read_fw(uncore, reg);
mcr &= ~mcr_slice_subslice_mask;
mcr |= default_mcr_s_ss_select;
- I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
+ intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
- intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
- spin_unlock_irq(&dev_priv->uncore.lock);
+ intel_uncore_forcewake_put__locked(uncore, fw_domains);
+ spin_unlock_irq(&uncore->lock);
return ret;
}
@@ -920,6 +962,7 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine,
struct intel_instdone *instdone)
{
struct drm_i915_private *dev_priv = engine->i915;
+ struct intel_uncore *uncore = engine->uncore;
u32 mmio_base = engine->mmio_base;
int slice;
int subslice;
@@ -928,12 +971,14 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine,
switch (INTEL_GEN(dev_priv)) {
default:
- instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
+ instdone->instdone =
+ intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
- if (engine->id != RCS)
+ if (engine->id != RCS0)
break;
- instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
+ instdone->slice_common =
+ intel_uncore_read(uncore, GEN7_SC_INSTDONE);
for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
instdone->sampler[slice][subslice] =
read_subslice_reg(dev_priv, slice, subslice,
@@ -944,28 +989,33 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine,
}
break;
case 7:
- instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
+ instdone->instdone =
+ intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
- if (engine->id != RCS)
+ if (engine->id != RCS0)
break;
- instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
- instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
- instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
+ instdone->slice_common =
+ intel_uncore_read(uncore, GEN7_SC_INSTDONE);
+ instdone->sampler[0][0] =
+ intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE);
+ instdone->row[0][0] =
+ intel_uncore_read(uncore, GEN7_ROW_INSTDONE);
break;
case 6:
case 5:
case 4:
- instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
-
- if (engine->id == RCS)
+ instdone->instdone =
+ intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
+ if (engine->id == RCS0)
/* HACK: Using the wrong struct member */
- instdone->slice_common = I915_READ(GEN4_INSTDONE1);
+ instdone->slice_common =
+ intel_uncore_read(uncore, GEN4_INSTDONE1);
break;
case 3:
case 2:
- instdone->instdone = I915_READ(GEN2_INSTDONE);
+ instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE);
break;
}
}
@@ -985,12 +1035,13 @@ static bool ring_is_idle(struct intel_engine_cs *engine)
return true;
/* First check that no commands are left in the ring */
- if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
- (I915_READ_TAIL(engine) & TAIL_ADDR))
+ if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) !=
+ (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR))
idle = false;
/* No bit for gen2, so assume the CS parser is idle */
- if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE))
+ if (INTEL_GEN(dev_priv) > 2 &&
+ !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE))
idle = false;
intel_runtime_pm_put(dev_priv, wakeref);
@@ -1007,16 +1058,10 @@ static bool ring_is_idle(struct intel_engine_cs *engine)
*/
bool intel_engine_is_idle(struct intel_engine_cs *engine)
{
- struct drm_i915_private *dev_priv = engine->i915;
-
/* More white lies, if wedged, hw state is inconsistent */
- if (i915_terminally_wedged(&dev_priv->gpu_error))
+ if (i915_reset_failed(engine->i915))
return true;
- /* Any inflight/incomplete requests? */
- if (!intel_engine_signaled(engine, intel_engine_last_submit(engine)))
- return false;
-
/* Waiting to drain ELSP? */
if (READ_ONCE(engine->execlists.active)) {
struct tasklet_struct *t = &engine->execlists.tasklet;
@@ -1045,7 +1090,7 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine)
return ring_is_idle(engine);
}
-bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
+bool intel_engines_are_idle(struct drm_i915_private *i915)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
@@ -1054,10 +1099,14 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
* If the driver is wedged, HW state may be very inconsistent and
* report that it is still busy, even though we have stopped using it.
*/
- if (i915_terminally_wedged(&dev_priv->gpu_error))
+ if (i915_reset_failed(i915))
return true;
- for_each_engine(engine, dev_priv, id) {
+ /* Already parked (and passed an idleness test); must still be idle */
+ if (!READ_ONCE(i915->gt.awake))
+ return true;
+
+ for_each_engine(engine, i915, id) {
if (!intel_engine_is_idle(engine))
return false;
}
@@ -1065,34 +1114,6 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
return true;
}
-/**
- * intel_engine_has_kernel_context:
- * @engine: the engine
- *
- * Returns true if the last context to be executed on this engine, or has been
- * executed if the engine is already idle, is the kernel context
- * (#i915.kernel_context).
- */
-bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine)
-{
- const struct intel_context *kernel_context =
- to_intel_context(engine->i915->kernel_context, engine);
- struct i915_request *rq;
-
- lockdep_assert_held(&engine->i915->drm.struct_mutex);
-
- /*
- * Check the last context seen by the engine. If active, it will be
- * the last request that remains in the timeline. When idle, it is
- * the last executed context as tracked by retirement.
- */
- rq = __i915_active_request_peek(&engine->timeline.last_request);
- if (rq)
- return rq->hw_context == kernel_context;
- else
- return engine->last_retired_context == kernel_context;
-}
-
void intel_engines_reset_default_submission(struct drm_i915_private *i915)
{
struct intel_engine_cs *engine;
@@ -1180,6 +1201,8 @@ void intel_engines_park(struct drm_i915_private *i915)
i915_gem_batch_pool_fini(&engine->batch_pool);
engine->execlists.no_priolist = false;
}
+
+ i915->gt.active_engines = 0;
}
/**
@@ -1283,15 +1306,14 @@ static void print_request(struct drm_printer *m,
x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf));
- drm_printf(m, "%s%x%s%s [%llx:%llx]%s @ %dms: %s\n",
+ drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
prefix,
- rq->global_seqno,
+ rq->fence.context, rq->fence.seqno,
i915_request_completed(rq) ? "!" :
i915_request_started(rq) ? "*" :
"",
test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
&rq->fence.flags) ? "+" : "",
- rq->fence.context, rq->fence.seqno,
buf,
jiffies_to_msecs(jiffies - rq->emitted_jiffies),
name);
@@ -1334,25 +1356,26 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
&engine->execlists;
u64 addr;
- if (engine->id == RCS && IS_GEN_RANGE(dev_priv, 4, 7))
- drm_printf(m, "\tCCID: 0x%08x\n", I915_READ(CCID));
+ if (engine->id == RCS0 && IS_GEN_RANGE(dev_priv, 4, 7))
+ drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID));
drm_printf(m, "\tRING_START: 0x%08x\n",
- I915_READ(RING_START(engine->mmio_base)));
+ ENGINE_READ(engine, RING_START));
drm_printf(m, "\tRING_HEAD: 0x%08x\n",
- I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR);
+ ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR);
drm_printf(m, "\tRING_TAIL: 0x%08x\n",
- I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR);
+ ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR);
drm_printf(m, "\tRING_CTL: 0x%08x%s\n",
- I915_READ(RING_CTL(engine->mmio_base)),
- I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
+ ENGINE_READ(engine, RING_CTL),
+ ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
if (INTEL_GEN(engine->i915) > 2) {
drm_printf(m, "\tRING_MODE: 0x%08x%s\n",
- I915_READ(RING_MI_MODE(engine->mmio_base)),
- I915_READ(RING_MI_MODE(engine->mmio_base)) & (MODE_IDLE) ? " [idle]" : "");
+ ENGINE_READ(engine, RING_MI_MODE),
+ ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : "");
}
if (INTEL_GEN(dev_priv) >= 6) {
- drm_printf(m, "\tRING_IMR: %08x\n", I915_READ_IMR(engine));
+ drm_printf(m, "\tRING_IMR: %08x\n",
+ ENGINE_READ(engine, RING_IMR));
}
addr = intel_engine_get_active_head(engine);
@@ -1362,57 +1385,53 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
upper_32_bits(addr), lower_32_bits(addr));
if (INTEL_GEN(dev_priv) >= 8)
- addr = I915_READ64_2x32(RING_DMA_FADD(engine->mmio_base),
- RING_DMA_FADD_UDW(engine->mmio_base));
+ addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW);
else if (INTEL_GEN(dev_priv) >= 4)
- addr = I915_READ(RING_DMA_FADD(engine->mmio_base));
+ addr = ENGINE_READ(engine, RING_DMA_FADD);
else
- addr = I915_READ(DMA_FADD_I8XX);
+ addr = ENGINE_READ(engine, DMA_FADD_I8XX);
drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
upper_32_bits(addr), lower_32_bits(addr));
if (INTEL_GEN(dev_priv) >= 4) {
drm_printf(m, "\tIPEIR: 0x%08x\n",
- I915_READ(RING_IPEIR(engine->mmio_base)));
+ ENGINE_READ(engine, RING_IPEIR));
drm_printf(m, "\tIPEHR: 0x%08x\n",
- I915_READ(RING_IPEHR(engine->mmio_base)));
+ ENGINE_READ(engine, RING_IPEHR));
} else {
- drm_printf(m, "\tIPEIR: 0x%08x\n", I915_READ(IPEIR));
- drm_printf(m, "\tIPEHR: 0x%08x\n", I915_READ(IPEHR));
+ drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR));
+ drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR));
}
if (HAS_EXECLISTS(dev_priv)) {
const u32 *hws =
&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
+ const u8 num_entries = execlists->csb_size;
unsigned int idx;
u8 read, write;
- drm_printf(m, "\tExeclist status: 0x%08x %08x\n",
- I915_READ(RING_EXECLIST_STATUS_LO(engine)),
- I915_READ(RING_EXECLIST_STATUS_HI(engine)));
+ drm_printf(m, "\tExeclist status: 0x%08x %08x, entries %u\n",
+ ENGINE_READ(engine, RING_EXECLIST_STATUS_LO),
+ ENGINE_READ(engine, RING_EXECLIST_STATUS_HI),
+ num_entries);
read = execlists->csb_head;
write = READ_ONCE(*execlists->csb_write);
- drm_printf(m, "\tExeclist CSB read %d, write %d [mmio:%d], tasklet queued? %s (%s)\n",
+ drm_printf(m, "\tExeclist CSB read %d, write %d, tasklet queued? %s (%s)\n",
read, write,
- GEN8_CSB_WRITE_PTR(I915_READ(RING_CONTEXT_STATUS_PTR(engine))),
yesno(test_bit(TASKLET_STATE_SCHED,
&engine->execlists.tasklet.state)),
enableddisabled(!atomic_read(&engine->execlists.tasklet.count)));
- if (read >= GEN8_CSB_ENTRIES)
+ if (read >= num_entries)
read = 0;
- if (write >= GEN8_CSB_ENTRIES)
+ if (write >= num_entries)
write = 0;
if (read > write)
- write += GEN8_CSB_ENTRIES;
+ write += num_entries;
while (read < write) {
- idx = ++read % GEN8_CSB_ENTRIES;
- drm_printf(m, "\tExeclist CSB[%d]: 0x%08x [mmio:0x%08x], context: %d [mmio:%d]\n",
- idx,
- hws[idx * 2],
- I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
- hws[idx * 2 + 1],
- I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)));
+ idx = ++read % num_entries;
+ drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
+ idx, hws[idx * 2], hws[idx * 2 + 1]);
}
rcu_read_lock();
@@ -1425,10 +1444,11 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
char hdr[80];
snprintf(hdr, sizeof(hdr),
- "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x}, rq: ",
+ "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x, seqno:%08x}, rq: ",
idx, count,
i915_ggtt_offset(rq->ring->vma),
- rq->timeline->hwsp_offset);
+ rq->timeline->hwsp_offset,
+ hwsp_seqno(rq));
print_request(m, rq, hdr);
} else {
drm_printf(m, "\t\tELSP[%d] idle\n", idx);
@@ -1438,11 +1458,11 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
rcu_read_unlock();
} else if (INTEL_GEN(dev_priv) > 6) {
drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
- I915_READ(RING_PP_DIR_BASE(engine)));
+ ENGINE_READ(engine, RING_PP_DIR_BASE));
drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
- I915_READ(RING_PP_DIR_BASE_READ(engine)));
+ ENGINE_READ(engine, RING_PP_DIR_BASE_READ));
drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
- I915_READ(RING_PP_DIR_DCLV(engine)));
+ ENGINE_READ(engine, RING_PP_DIR_DCLV));
}
}
@@ -1495,13 +1515,12 @@ void intel_engine_dump(struct intel_engine_cs *engine,
va_end(ap);
}
- if (i915_terminally_wedged(&engine->i915->gpu_error))
+ if (i915_reset_failed(engine->i915))
drm_printf(m, "*** WEDGED ***\n");
- drm_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [%d ms]\n",
- intel_engine_get_seqno(engine),
- intel_engine_last_submit(engine),
- engine->hangcheck.seqno,
+ drm_printf(m, "\tHangcheck %x:%x [%d ms]\n",
+ engine->hangcheck.last_seqno,
+ engine->hangcheck.next_seqno,
jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp));
drm_printf(m, "\tReset count: %d (global %d)\n",
i915_reset_engine_count(error, engine),
@@ -1521,7 +1540,7 @@ void intel_engine_dump(struct intel_engine_cs *engine,
if (&rq->link != &engine->timeline.requests)
print_request(m, rq, "\t\tlast ");
- rq = i915_gem_find_active_request(engine);
+ rq = intel_engine_find_active_request(engine);
if (rq) {
print_request(m, rq, "\t\tactive ");
@@ -1688,6 +1707,50 @@ void intel_disable_engine_stats(struct intel_engine_cs *engine)
write_sequnlock_irqrestore(&engine->stats.lock, flags);
}
+static bool match_ring(struct i915_request *rq)
+{
+ u32 ring = ENGINE_READ(rq->engine, RING_START);
+
+ return ring == i915_ggtt_offset(rq->ring->vma);
+}
+
+struct i915_request *
+intel_engine_find_active_request(struct intel_engine_cs *engine)
+{
+ struct i915_request *request, *active = NULL;
+ unsigned long flags;
+
+ /*
+ * We are called by the error capture, reset and to dump engine
+ * state at random points in time. In particular, note that neither is
+ * crucially ordered with an interrupt. After a hang, the GPU is dead
+ * and we assume that no more writes can happen (we waited long enough
+ * for all writes that were in transaction to be flushed) - adding an
+ * extra delay for a recent interrupt is pointless. Hence, we do
+ * not need an engine->irq_seqno_barrier() before the seqno reads.
+ * At all other times, we must assume the GPU is still running, but
+ * we only care about the snapshot of this moment.
+ */
+ spin_lock_irqsave(&engine->timeline.lock, flags);
+ list_for_each_entry(request, &engine->timeline.requests, link) {
+ if (i915_request_completed(request))
+ continue;
+
+ if (!i915_request_started(request))
+ break;
+
+ /* More than one preemptible request may match! */
+ if (!match_ring(request))
+ break;
+
+ active = request;
+ break;
+ }
+ spin_unlock_irqrestore(&engine->timeline.lock, flags);
+
+ return active;
+}
+
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/mock_engine.c"
#include "selftests/intel_engine_cs.c"