aboutsummaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/i915/i915_request.h
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/i915_request.h')
-rw-r--r--drivers/gpu/drm/i915/i915_request.h397
1 files changed, 344 insertions, 53 deletions
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index edbbdfec24ab..28b1f9db5487 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -26,28 +26,48 @@
#define I915_REQUEST_H
#include <linux/dma-fence.h>
+#include <linux/hrtimer.h>
+#include <linux/irq_work.h>
+#include <linux/llist.h>
#include <linux/lockdep.h>
+#include "gem/i915_gem_context_types.h"
+#include "gt/intel_context_types.h"
#include "gt/intel_engine_types.h"
+#include "gt/intel_timeline_types.h"
#include "i915_gem.h"
#include "i915_scheduler.h"
#include "i915_selftest.h"
#include "i915_sw_fence.h"
+#include "i915_vma_resource.h"
#include <uapi/drm/i915_drm.h>
struct drm_file;
struct drm_i915_gem_object;
+struct drm_printer;
+struct i915_deps;
struct i915_request;
-struct i915_timeline;
-struct i915_timeline_cacheline;
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
struct i915_capture_list {
+ struct i915_vma_resource *vma_res;
struct i915_capture_list *next;
- struct i915_vma *vma;
};
+void i915_request_free_capture_list(struct i915_capture_list *capture);
+#else
+#define i915_request_free_capture_list(_a) do {} while (0)
+#endif
+
+#define RQ_TRACE(rq, fmt, ...) do { \
+ const struct i915_request *rq__ = (rq); \
+ ENGINE_TRACE(rq__->engine, "fence %llx:%lld, current %d " fmt, \
+ rq__->fence.context, rq__->fence.seqno, \
+ hwsp_seqno(rq__), ##__VA_ARGS__); \
+} while (0)
+
enum {
/*
* I915_FENCE_FLAG_ACTIVE - this request is currently submitted to HW.
@@ -63,12 +83,93 @@ enum {
I915_FENCE_FLAG_ACTIVE = DMA_FENCE_FLAG_USER_BITS,
/*
+ * I915_FENCE_FLAG_PQUEUE - this request is ready for execution
+ *
+ * Using the scheduler, when a request is ready for execution it is put
+ * into the priority queue, and removed from that queue when transferred
+ * to the HW runlists. We want to track its membership within the
+ * priority queue so that we can easily check before rescheduling.
+ *
+ * See i915_request_in_priority_queue()
+ */
+ I915_FENCE_FLAG_PQUEUE,
+
+ /*
+ * I915_FENCE_FLAG_HOLD - this request is currently on hold
+ *
+ * This request has been suspended, pending an ongoing investigation.
+ */
+ I915_FENCE_FLAG_HOLD,
+
+ /*
+ * I915_FENCE_FLAG_INITIAL_BREADCRUMB - this request has the initial
+ * breadcrumb that marks the end of semaphore waits and start of the
+ * user payload.
+ */
+ I915_FENCE_FLAG_INITIAL_BREADCRUMB,
+
+ /*
* I915_FENCE_FLAG_SIGNAL - this request is currently on signal_list
*
* Internal bookkeeping used by the breadcrumb code to track when
* a request is on the various signal_list.
*/
I915_FENCE_FLAG_SIGNAL,
+
+ /*
+ * I915_FENCE_FLAG_NOPREEMPT - this request should not be preempted
+ *
+ * The execution of some requests should not be interrupted. This is
+ * a sensitive operation as it makes the request super important,
+ * blocking other higher priority work. Abuse of this flag will
+ * lead to quality of service issues.
+ */
+ I915_FENCE_FLAG_NOPREEMPT,
+
+ /*
+ * I915_FENCE_FLAG_SENTINEL - this request should be last in the queue
+ *
+ * A high priority sentinel request may be submitted to clear the
+ * submission queue. As it will be the only request in-flight, upon
+ * execution all other active requests will have been preempted and
+ * unsubmitted. This preemptive pulse is used to re-evaluate the
+ * in-flight requests, particularly in cases where an active context
+ * is banned and those active requests need to be cancelled.
+ */
+ I915_FENCE_FLAG_SENTINEL,
+
+ /*
+ * I915_FENCE_FLAG_BOOST - upclock the gpu for this request
+ *
+ * Some requests are more important than others! In particular, a
+ * request that the user is waiting on is typically required for
+ * interactive latency, for which we want to minimise by upclocking
+ * the GPU. Here we track such boost requests on a per-request basis.
+ */
+ I915_FENCE_FLAG_BOOST,
+
+ /*
+ * I915_FENCE_FLAG_SUBMIT_PARALLEL - request with a context in a
+ * parent-child relationship (parallel submission, multi-lrc) should
+ * trigger a submission to the GuC rather than just moving the context
+ * tail.
+ */
+ I915_FENCE_FLAG_SUBMIT_PARALLEL,
+
+ /*
+ * I915_FENCE_FLAG_SKIP_PARALLEL - request with a context in a
+ * parent-child relationship (parallel submission, multi-lrc) that
+ * hit an error while generating requests in the execbuf IOCTL.
+ * Indicates this request should be skipped as another request in
+ * submission / relationship encoutered an error.
+ */
+ I915_FENCE_FLAG_SKIP_PARALLEL,
+
+ /*
+ * I915_FENCE_FLAG_COMPOSITE - Indicates fence is part of a composite
+ * fence (dma_fence_array) and i915 generated for parallel submission.
+ */
+ I915_FENCE_FLAG_COMPOSITE,
};
/**
@@ -95,9 +196,6 @@ struct i915_request {
struct dma_fence fence;
spinlock_t lock;
- /** On Which ring this request was generated */
- struct drm_i915_private *i915;
-
/**
* Context and ring buffer related to this request
* Contexts are refcounted, so when this request is associated with a
@@ -108,12 +206,13 @@ struct i915_request {
* i915_request_free() will then decrement the refcount on the
* context.
*/
- struct i915_gem_context *gem_context;
struct intel_engine_cs *engine;
- struct intel_context *hw_context;
+ struct intel_context *context;
struct intel_ring *ring;
- struct i915_timeline *timeline;
+ struct intel_timeline __rcu *timeline;
+
struct list_head signal_link;
+ struct llist_node signal_node;
/*
* The rcu epoch of when this request was allocated. Used to judiciously
@@ -143,9 +242,18 @@ struct i915_request {
union {
wait_queue_entry_t submitq;
struct i915_sw_dma_fence_cb dmaq;
+ struct i915_request_duration_cb {
+ struct dma_fence_cb cb;
+ ktime_t emitted;
+ } duration;
};
- struct list_head execute_cb;
+ struct llist_head execute_cb;
struct i915_sw_fence semaphore;
+ /**
+ * @submit_work: complete submit fence from an IRQ if needed for
+ * locking hierarchy reasons.
+ */
+ struct irq_work submit_work;
/*
* A list of everyone we wait upon, and everyone who waits upon us.
@@ -167,16 +275,6 @@ struct i915_request {
*/
const u32 *hwsp_seqno;
- /*
- * If we need to access the timeline's seqno for this request in
- * another request, we need to keep a read reference to this associated
- * cacheline, so that we do not free and recycle it before the foreign
- * observers have completed. Hence, we keep a pointer to the cacheline
- * inside the timeline's HWSP vma, but it is only valid while this
- * request has not completed and guarded by the timeline mutex.
- */
- struct i915_timeline_cacheline *hwsp_cacheline;
-
/** Position in the ring of the start of the request */
u32 head;
@@ -199,10 +297,12 @@ struct i915_request {
/** Preallocate space in the ring for the emitting the request */
u32 reserved_space;
- /** Batch buffer related to this request if any (used for
- * error state dump only).
- */
- struct i915_vma *batch;
+ /** Batch buffer pointer for selftest internal use. */
+ I915_SELFTEST_DECLARE(struct i915_vma *batch);
+
+ struct i915_vma_resource *batch_res;
+
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
/**
* Additional buffers requested by userspace to be captured upon
* a GPU hang. The vma/obj on this list are protected by their
@@ -210,22 +310,41 @@ struct i915_request {
* on the active_list (of their final request).
*/
struct i915_capture_list *capture_list;
- struct list_head active_list;
+#endif
/** Time at which this request was emitted, in jiffies. */
unsigned long emitted_jiffies;
- bool waitboost;
-
/** timeline->request entry for this request */
struct list_head link;
- /** ring->request_list entry for this request */
- struct list_head ring_link;
+ /** Watchdog support fields. */
+ struct i915_request_watchdog {
+ struct llist_node link;
+ struct hrtimer timer;
+ } watchdog;
+
+ /**
+ * @guc_fence_link: Requests may need to be stalled when using GuC
+ * submission waiting for certain GuC operations to complete. If that is
+ * the case, stalled requests are added to a per context list of stalled
+ * requests. The below list_head is the link in that list. Protected by
+ * ce->guc_state.lock.
+ */
+ struct list_head guc_fence_link;
- struct drm_i915_file_private *file_priv;
- /** file_priv list entry for this request */
- struct list_head client_link;
+ /**
+ * @guc_prio: Priority level while the request is in flight. Differs
+ * from i915 scheduler priority. See comment above
+ * I915_SCHEDULER_CAP_STATIC_PRIORITY_MAP for details. Protected by
+ * ce->guc_active.lock. Two special values (GUC_PRIO_INIT and
+ * GUC_PRIO_FINI) outside the GuC priority range are used to indicate
+ * if the priority has not been initialized yet or if no more updates
+ * are possible because the request has completed.
+ */
+#define GUC_PRIO_INIT 0xff
+#define GUC_PRIO_FINI 0xfe
+ u8 guc_prio;
I915_SELFTEST_DECLARE(struct {
struct list_head link;
@@ -242,13 +361,23 @@ static inline bool dma_fence_is_i915(const struct dma_fence *fence)
return fence->ops == &i915_fence_ops;
}
+struct kmem_cache *i915_request_slab_cache(void);
+
struct i915_request * __must_check
__i915_request_create(struct intel_context *ce, gfp_t gfp);
struct i915_request * __must_check
i915_request_create(struct intel_context *ce);
+void __i915_request_skip(struct i915_request *rq);
+bool i915_request_set_error_once(struct i915_request *rq, int error);
+struct i915_request *i915_request_mark_eio(struct i915_request *rq);
+
struct i915_request *__i915_request_commit(struct i915_request *request);
+void __i915_request_queue(struct i915_request *rq,
+ const struct i915_sched_attr *attr);
+void __i915_request_queue_bh(struct i915_request *rq);
+bool i915_request_retire(struct i915_request *rq);
void i915_request_retire_upto(struct i915_request *rq);
static inline struct i915_request *
@@ -283,34 +412,37 @@ int i915_request_await_object(struct i915_request *to,
bool write);
int i915_request_await_dma_fence(struct i915_request *rq,
struct dma_fence *fence);
+int i915_request_await_deps(struct i915_request *rq, const struct i915_deps *deps);
int i915_request_await_execution(struct i915_request *rq,
- struct dma_fence *fence,
- void (*hook)(struct i915_request *rq,
- struct dma_fence *signal));
+ struct dma_fence *fence);
void i915_request_add(struct i915_request *rq);
-void __i915_request_submit(struct i915_request *request);
+bool __i915_request_submit(struct i915_request *request);
void i915_request_submit(struct i915_request *request);
-void i915_request_skip(struct i915_request *request, int error);
-
void __i915_request_unsubmit(struct i915_request *request);
void i915_request_unsubmit(struct i915_request *request);
-/* Note: part of the intel_breadcrumbs family */
-bool i915_request_enable_breadcrumb(struct i915_request *request);
-void i915_request_cancel_breadcrumb(struct i915_request *request);
+void i915_request_cancel(struct i915_request *rq, int error);
+
+long i915_request_wait_timeout(struct i915_request *rq,
+ unsigned int flags,
+ long timeout)
+ __attribute__((nonnull(1)));
long i915_request_wait(struct i915_request *rq,
unsigned int flags,
long timeout)
__attribute__((nonnull(1)));
#define I915_WAIT_INTERRUPTIBLE BIT(0)
-#define I915_WAIT_LOCKED BIT(1) /* struct_mutex held, handle GPU reset */
-#define I915_WAIT_PRIORITY BIT(2) /* small priority bump for the request */
-#define I915_WAIT_ALL BIT(3) /* used by i915_gem_object_wait() */
-#define I915_WAIT_FOR_IDLE_BOOST BIT(4)
+#define I915_WAIT_PRIORITY BIT(1) /* small priority bump for the request */
+#define I915_WAIT_ALL BIT(2) /* used by i915_gem_object_wait() */
+
+void i915_request_show(struct drm_printer *m,
+ const struct i915_request *rq,
+ const char *prefix,
+ int indent);
static inline bool i915_request_signaled(const struct i915_request *rq)
{
@@ -323,6 +455,17 @@ static inline bool i915_request_is_active(const struct i915_request *rq)
return test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
}
+static inline bool i915_request_in_priority_queue(const struct i915_request *rq)
+{
+ return test_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
+}
+
+static inline bool
+i915_request_has_initial_breadcrumb(const struct i915_request *rq)
+{
+ return test_bit(I915_FENCE_FLAG_INITIAL_BREADCRUMB, &rq->fence.flags);
+}
+
/**
* Returns true if seq1 is later than seq2.
*/
@@ -333,7 +476,9 @@ static inline bool i915_seqno_passed(u32 seq1, u32 seq2)
static inline u32 __hwsp_seqno(const struct i915_request *rq)
{
- return READ_ONCE(*rq->hwsp_seqno);
+ const u32 *hwsp = READ_ONCE(rq->hwsp_seqno);
+
+ return READ_ONCE(*hwsp);
}
/**
@@ -362,7 +507,7 @@ static inline u32 hwsp_seqno(const struct i915_request *rq)
static inline bool __i915_request_has_started(const struct i915_request *rq)
{
- return i915_seqno_passed(hwsp_seqno(rq), rq->fence.seqno - 1);
+ return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno - 1);
}
/**
@@ -393,11 +538,19 @@ static inline bool __i915_request_has_started(const struct i915_request *rq)
*/
static inline bool i915_request_started(const struct i915_request *rq)
{
+ bool result;
+
if (i915_request_signaled(rq))
return true;
- /* Remember: started but may have since been preempted! */
- return __i915_request_has_started(rq);
+ result = true;
+ rcu_read_lock(); /* the HWSP may be freed at runtime */
+ if (likely(!i915_request_signaled(rq)))
+ /* Remember: started but may have since been preempted! */
+ result = __i915_request_has_started(rq);
+ rcu_read_unlock();
+
+ return result;
}
/**
@@ -410,25 +563,163 @@ static inline bool i915_request_started(const struct i915_request *rq)
*/
static inline bool i915_request_is_running(const struct i915_request *rq)
{
+ bool result;
+
if (!i915_request_is_active(rq))
return false;
- return __i915_request_has_started(rq);
+ rcu_read_lock();
+ result = __i915_request_has_started(rq) && i915_request_is_active(rq);
+ rcu_read_unlock();
+
+ return result;
+}
+
+/**
+ * i915_request_is_ready - check if the request is ready for execution
+ * @rq: the request
+ *
+ * Upon construction, the request is instructed to wait upon various
+ * signals before it is ready to be executed by the HW. That is, we do
+ * not want to start execution and read data before it is written. In practice,
+ * this is controlled with a mixture of interrupts and semaphores. Once
+ * the submit fence is completed, the backend scheduler will place the
+ * request into its queue and from there submit it for execution. So we
+ * can detect when a request is eligible for execution (and is under control
+ * of the scheduler) by querying where it is in any of the scheduler's lists.
+ *
+ * Returns true if the request is ready for execution (it may be inflight),
+ * false otherwise.
+ */
+static inline bool i915_request_is_ready(const struct i915_request *rq)
+{
+ return !list_empty(&rq->sched.link);
+}
+
+static inline bool __i915_request_is_complete(const struct i915_request *rq)
+{
+ return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
}
static inline bool i915_request_completed(const struct i915_request *rq)
{
+ bool result;
+
if (i915_request_signaled(rq))
return true;
- return i915_seqno_passed(hwsp_seqno(rq), rq->fence.seqno);
+ result = true;
+ rcu_read_lock(); /* the HWSP may be freed at runtime */
+ if (likely(!i915_request_signaled(rq)))
+ result = __i915_request_is_complete(rq);
+ rcu_read_unlock();
+
+ return result;
}
static inline void i915_request_mark_complete(struct i915_request *rq)
{
- rq->hwsp_seqno = (u32 *)&rq->fence.seqno; /* decouple from HWSP */
+ WRITE_ONCE(rq->hwsp_seqno, /* decouple from HWSP */
+ (u32 *)&rq->fence.seqno);
+}
+
+static inline bool i915_request_has_waitboost(const struct i915_request *rq)
+{
+ return test_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags);
+}
+
+static inline bool i915_request_has_nopreempt(const struct i915_request *rq)
+{
+ /* Preemption should only be disabled very rarely */
+ return unlikely(test_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags));
+}
+
+static inline bool i915_request_has_sentinel(const struct i915_request *rq)
+{
+ return unlikely(test_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags));
+}
+
+static inline bool i915_request_on_hold(const struct i915_request *rq)
+{
+ return unlikely(test_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags));
+}
+
+static inline void i915_request_set_hold(struct i915_request *rq)
+{
+ set_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags);
+}
+
+static inline void i915_request_clear_hold(struct i915_request *rq)
+{
+ clear_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags);
+}
+
+static inline struct intel_timeline *
+i915_request_timeline(const struct i915_request *rq)
+{
+ /* Valid only while the request is being constructed (or retired). */
+ return rcu_dereference_protected(rq->timeline,
+ lockdep_is_held(&rcu_access_pointer(rq->timeline)->mutex) ||
+ test_bit(CONTEXT_IS_PARKING, &rq->context->flags));
+}
+
+static inline struct i915_gem_context *
+i915_request_gem_context(const struct i915_request *rq)
+{
+ /* Valid only while the request is being constructed (or retired). */
+ return rcu_dereference_protected(rq->context->gem_context, true);
+}
+
+static inline struct intel_timeline *
+i915_request_active_timeline(const struct i915_request *rq)
+{
+ /*
+ * When in use during submission, we are protected by a guarantee that
+ * the context/timeline is pinned and must remain pinned until after
+ * this submission.
+ */
+ return rcu_dereference_protected(rq->timeline,
+ lockdep_is_held(&rq->engine->sched_engine->lock));
+}
+
+static inline u32
+i915_request_active_seqno(const struct i915_request *rq)
+{
+ u32 hwsp_phys_base =
+ page_mask_bits(i915_request_active_timeline(rq)->hwsp_offset);
+ u32 hwsp_relative_offset = offset_in_page(rq->hwsp_seqno);
+
+ /*
+ * Because of wraparound, we cannot simply take tl->hwsp_offset,
+ * but instead use the fact that the relative for vaddr is the
+ * offset as for hwsp_offset. Take the top bits from tl->hwsp_offset
+ * and combine them with the relative offset in rq->hwsp_seqno.
+ *
+ * As rw->hwsp_seqno is rewritten when signaled, this only works
+ * when the request isn't signaled yet, but at that point you
+ * no longer need the offset.
+ */
+
+ return hwsp_phys_base + hwsp_relative_offset;
}
-bool i915_retire_requests(struct drm_i915_private *i915);
+bool
+i915_request_active_engine(struct i915_request *rq,
+ struct intel_engine_cs **active);
+
+void i915_request_notify_execute_cb_imm(struct i915_request *rq);
+
+enum i915_request_state {
+ I915_REQUEST_UNKNOWN = 0,
+ I915_REQUEST_COMPLETE,
+ I915_REQUEST_PENDING,
+ I915_REQUEST_QUEUED,
+ I915_REQUEST_ACTIVE,
+};
+
+enum i915_request_state i915_test_request_state(struct i915_request *rq);
+
+void i915_request_module_exit(void);
+int i915_request_module_init(void);
#endif /* I915_REQUEST_H */