blob: a041492fdd4629f8e4ad01d882473121c6ac6824 [file] [log] [blame]
Eric Anholt62fdfea2010-05-21 13:26:39 -07001/*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
30#include "drmP.h"
31#include "drm.h"
Eric Anholt62fdfea2010-05-21 13:26:39 -070032#include "i915_drv.h"
Zou Nan hai8187a2b2010-05-21 09:08:55 +080033#include "i915_drm.h"
Eric Anholt62fdfea2010-05-21 13:26:39 -070034#include "i915_trace.h"
Xiang, Haihao881f47b2010-09-19 14:40:43 +010035#include "intel_drv.h"
Eric Anholt62fdfea2010-05-21 13:26:39 -070036
Jesse Barnes8d315282011-10-16 10:23:31 +020037/*
38 * 965+ support PIPE_CONTROL commands, which provide finer grained control
39 * over cache flushing.
40 */
41struct pipe_control {
42 struct drm_i915_gem_object *obj;
43 volatile u32 *cpu_page;
44 u32 gtt_offset;
45};
46
Chris Wilsonc7dca472011-01-20 17:00:10 +000047static inline int ring_space(struct intel_ring_buffer *ring)
48{
49 int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
50 if (space < 0)
51 space += ring->size;
52 return space;
53}
54
Chris Wilsonb72f3ac2011-01-04 17:34:02 +000055static int
Chris Wilson46f0f8d2012-04-18 11:12:11 +010056gen2_render_ring_flush(struct intel_ring_buffer *ring,
57 u32 invalidate_domains,
58 u32 flush_domains)
59{
60 u32 cmd;
61 int ret;
62
63 cmd = MI_FLUSH;
Daniel Vetter31b14c92012-04-19 16:45:22 +020064 if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
Chris Wilson46f0f8d2012-04-18 11:12:11 +010065 cmd |= MI_NO_WRITE_FLUSH;
66
67 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
68 cmd |= MI_READ_FLUSH;
69
70 ret = intel_ring_begin(ring, 2);
71 if (ret)
72 return ret;
73
74 intel_ring_emit(ring, cmd);
75 intel_ring_emit(ring, MI_NOOP);
76 intel_ring_advance(ring);
77
78 return 0;
79}
80
81static int
82gen4_render_ring_flush(struct intel_ring_buffer *ring,
83 u32 invalidate_domains,
84 u32 flush_domains)
Eric Anholt62fdfea2010-05-21 13:26:39 -070085{
Chris Wilson78501ea2010-10-27 12:18:21 +010086 struct drm_device *dev = ring->dev;
Chris Wilson6f392d52010-08-07 11:01:22 +010087 u32 cmd;
Chris Wilsonb72f3ac2011-01-04 17:34:02 +000088 int ret;
Chris Wilson6f392d52010-08-07 11:01:22 +010089
Chris Wilson36d527d2011-03-19 22:26:49 +000090 /*
91 * read/write caches:
92 *
93 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
94 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
95 * also flushed at 2d versus 3d pipeline switches.
96 *
97 * read-only caches:
98 *
99 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
100 * MI_READ_FLUSH is set, and is always flushed on 965.
101 *
102 * I915_GEM_DOMAIN_COMMAND may not exist?
103 *
104 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
105 * invalidated when MI_EXE_FLUSH is set.
106 *
107 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
108 * invalidated with every MI_FLUSH.
109 *
110 * TLBs:
111 *
112 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
113 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
114 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
115 * are flushed at any MI_FLUSH.
116 */
117
118 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
Chris Wilson46f0f8d2012-04-18 11:12:11 +0100119 if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
Chris Wilson36d527d2011-03-19 22:26:49 +0000120 cmd &= ~MI_NO_WRITE_FLUSH;
Chris Wilson36d527d2011-03-19 22:26:49 +0000121 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
122 cmd |= MI_EXE_FLUSH;
123
124 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
125 (IS_G4X(dev) || IS_GEN5(dev)))
126 cmd |= MI_INVALIDATE_ISP;
127
128 ret = intel_ring_begin(ring, 2);
129 if (ret)
130 return ret;
131
132 intel_ring_emit(ring, cmd);
133 intel_ring_emit(ring, MI_NOOP);
134 intel_ring_advance(ring);
Chris Wilsonb72f3ac2011-01-04 17:34:02 +0000135
136 return 0;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800137}
138
Jesse Barnes8d315282011-10-16 10:23:31 +0200139/**
140 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
141 * implementing two workarounds on gen6. From section 1.4.7.1
142 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
143 *
144 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
145 * produced by non-pipelined state commands), software needs to first
146 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
147 * 0.
148 *
149 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
150 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
151 *
152 * And the workaround for these two requires this workaround first:
153 *
154 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
155 * BEFORE the pipe-control with a post-sync op and no write-cache
156 * flushes.
157 *
158 * And this last workaround is tricky because of the requirements on
159 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
160 * volume 2 part 1:
161 *
162 * "1 of the following must also be set:
163 * - Render Target Cache Flush Enable ([12] of DW1)
164 * - Depth Cache Flush Enable ([0] of DW1)
165 * - Stall at Pixel Scoreboard ([1] of DW1)
166 * - Depth Stall ([13] of DW1)
167 * - Post-Sync Operation ([13] of DW1)
168 * - Notify Enable ([8] of DW1)"
169 *
170 * The cache flushes require the workaround flush that triggered this
171 * one, so we can't use it. Depth stall would trigger the same.
172 * Post-sync nonzero is what triggered this second workaround, so we
173 * can't use that one either. Notify enable is IRQs, which aren't
174 * really our business. That leaves only stall at scoreboard.
175 */
176static int
177intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
178{
179 struct pipe_control *pc = ring->private;
180 u32 scratch_addr = pc->gtt_offset + 128;
181 int ret;
182
183
184 ret = intel_ring_begin(ring, 6);
185 if (ret)
186 return ret;
187
188 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
189 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
190 PIPE_CONTROL_STALL_AT_SCOREBOARD);
191 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
192 intel_ring_emit(ring, 0); /* low dword */
193 intel_ring_emit(ring, 0); /* high dword */
194 intel_ring_emit(ring, MI_NOOP);
195 intel_ring_advance(ring);
196
197 ret = intel_ring_begin(ring, 6);
198 if (ret)
199 return ret;
200
201 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
202 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
203 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
204 intel_ring_emit(ring, 0);
205 intel_ring_emit(ring, 0);
206 intel_ring_emit(ring, MI_NOOP);
207 intel_ring_advance(ring);
208
209 return 0;
210}
211
212static int
213gen6_render_ring_flush(struct intel_ring_buffer *ring,
214 u32 invalidate_domains, u32 flush_domains)
215{
216 u32 flags = 0;
217 struct pipe_control *pc = ring->private;
218 u32 scratch_addr = pc->gtt_offset + 128;
219 int ret;
220
221 /* Force SNB workarounds for PIPE_CONTROL flushes */
222 intel_emit_post_sync_nonzero_flush(ring);
223
224 /* Just flush everything. Experiments have shown that reducing the
225 * number of bits based on the write domains has little performance
226 * impact.
227 */
228 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
Ben Widawskycc0f6392012-06-04 14:42:49 -0700229 flags |= PIPE_CONTROL_TLB_INVALIDATE;
Jesse Barnes8d315282011-10-16 10:23:31 +0200230 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
231 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
232 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
233 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
234 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
235 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
236
237 ret = intel_ring_begin(ring, 6);
238 if (ret)
239 return ret;
240
241 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
242 intel_ring_emit(ring, flags);
243 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
244 intel_ring_emit(ring, 0); /* lower dword */
245 intel_ring_emit(ring, 0); /* uppwer dword */
246 intel_ring_emit(ring, MI_NOOP);
247 intel_ring_advance(ring);
248
249 return 0;
250}
251
Chris Wilson78501ea2010-10-27 12:18:21 +0100252static void ring_write_tail(struct intel_ring_buffer *ring,
Chris Wilson297b0c52010-10-22 17:02:41 +0100253 u32 value)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800254{
Chris Wilson78501ea2010-10-27 12:18:21 +0100255 drm_i915_private_t *dev_priv = ring->dev->dev_private;
Chris Wilson297b0c52010-10-22 17:02:41 +0100256 I915_WRITE_TAIL(ring, value);
Xiang, Haihaod46eefa2010-09-16 10:43:12 +0800257}
258
Chris Wilson78501ea2010-10-27 12:18:21 +0100259u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800260{
Chris Wilson78501ea2010-10-27 12:18:21 +0100261 drm_i915_private_t *dev_priv = ring->dev->dev_private;
262 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
Daniel Vetter3d281d82010-09-24 21:14:22 +0200263 RING_ACTHD(ring->mmio_base) : ACTHD;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800264
265 return I915_READ(acthd_reg);
266}
267
Chris Wilson78501ea2010-10-27 12:18:21 +0100268static int init_ring_common(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800269{
Chris Wilson78501ea2010-10-27 12:18:21 +0100270 drm_i915_private_t *dev_priv = ring->dev->dev_private;
Chris Wilson05394f32010-11-08 19:18:58 +0000271 struct drm_i915_gem_object *obj = ring->obj;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800272 u32 head;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800273
274 /* Stop the ring if it's running. */
Daniel Vetter7f2ab692010-08-02 17:06:59 +0200275 I915_WRITE_CTL(ring, 0);
Daniel Vetter570ef602010-08-02 17:06:23 +0200276 I915_WRITE_HEAD(ring, 0);
Chris Wilson78501ea2010-10-27 12:18:21 +0100277 ring->write_tail(ring, 0);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800278
279 /* Initialize the ring. */
Chris Wilson05394f32010-11-08 19:18:58 +0000280 I915_WRITE_START(ring, obj->gtt_offset);
Daniel Vetter570ef602010-08-02 17:06:23 +0200281 head = I915_READ_HEAD(ring) & HEAD_ADDR;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800282
283 /* G45 ring initialization fails to reset head to zero */
284 if (head != 0) {
Chris Wilson6fd0d562010-12-05 20:42:33 +0000285 DRM_DEBUG_KMS("%s head not reset to zero "
286 "ctl %08x head %08x tail %08x start %08x\n",
287 ring->name,
288 I915_READ_CTL(ring),
289 I915_READ_HEAD(ring),
290 I915_READ_TAIL(ring),
291 I915_READ_START(ring));
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800292
Daniel Vetter570ef602010-08-02 17:06:23 +0200293 I915_WRITE_HEAD(ring, 0);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800294
Chris Wilson6fd0d562010-12-05 20:42:33 +0000295 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
296 DRM_ERROR("failed to set %s head to zero "
297 "ctl %08x head %08x tail %08x start %08x\n",
298 ring->name,
299 I915_READ_CTL(ring),
300 I915_READ_HEAD(ring),
301 I915_READ_TAIL(ring),
302 I915_READ_START(ring));
303 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700304 }
305
Daniel Vetter7f2ab692010-08-02 17:06:59 +0200306 I915_WRITE_CTL(ring,
Chris Wilsonae69b422010-11-07 11:45:52 +0000307 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
Chris Wilson5d031e52012-02-08 13:34:13 +0000308 | RING_VALID);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800309
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800310 /* If the head is still not zero, the ring is dead */
Sean Paulf01db982012-03-16 12:43:22 -0400311 if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
312 I915_READ_START(ring) == obj->gtt_offset &&
313 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
Chris Wilsone74cfed2010-11-09 10:16:56 +0000314 DRM_ERROR("%s initialization failed "
315 "ctl %08x head %08x tail %08x start %08x\n",
316 ring->name,
317 I915_READ_CTL(ring),
318 I915_READ_HEAD(ring),
319 I915_READ_TAIL(ring),
320 I915_READ_START(ring));
321 return -EIO;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800322 }
323
Chris Wilson78501ea2010-10-27 12:18:21 +0100324 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
325 i915_kernel_lost_context(ring->dev);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800326 else {
Chris Wilsonc7dca472011-01-20 17:00:10 +0000327 ring->head = I915_READ_HEAD(ring);
Daniel Vetter870e86d2010-08-02 16:29:44 +0200328 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
Chris Wilsonc7dca472011-01-20 17:00:10 +0000329 ring->space = ring_space(ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800330 }
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000331
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800332 return 0;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700333}
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800334
Chris Wilsonc6df5412010-12-15 09:56:50 +0000335static int
336init_pipe_control(struct intel_ring_buffer *ring)
337{
338 struct pipe_control *pc;
339 struct drm_i915_gem_object *obj;
340 int ret;
341
342 if (ring->private)
343 return 0;
344
345 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
346 if (!pc)
347 return -ENOMEM;
348
349 obj = i915_gem_alloc_object(ring->dev, 4096);
350 if (obj == NULL) {
351 DRM_ERROR("Failed to allocate seqno page\n");
352 ret = -ENOMEM;
353 goto err;
354 }
Chris Wilsone4ffd172011-04-04 09:44:39 +0100355
356 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
Chris Wilsonc6df5412010-12-15 09:56:50 +0000357
358 ret = i915_gem_object_pin(obj, 4096, true);
359 if (ret)
360 goto err_unref;
361
362 pc->gtt_offset = obj->gtt_offset;
363 pc->cpu_page = kmap(obj->pages[0]);
364 if (pc->cpu_page == NULL)
365 goto err_unpin;
366
367 pc->obj = obj;
368 ring->private = pc;
369 return 0;
370
371err_unpin:
372 i915_gem_object_unpin(obj);
373err_unref:
374 drm_gem_object_unreference(&obj->base);
375err:
376 kfree(pc);
377 return ret;
378}
379
380static void
381cleanup_pipe_control(struct intel_ring_buffer *ring)
382{
383 struct pipe_control *pc = ring->private;
384 struct drm_i915_gem_object *obj;
385
386 if (!ring->private)
387 return;
388
389 obj = pc->obj;
390 kunmap(obj->pages[0]);
391 i915_gem_object_unpin(obj);
392 drm_gem_object_unreference(&obj->base);
393
394 kfree(pc);
395 ring->private = NULL;
396}
397
Chris Wilson78501ea2010-10-27 12:18:21 +0100398static int init_render_ring(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800399{
Chris Wilson78501ea2010-10-27 12:18:21 +0100400 struct drm_device *dev = ring->dev;
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000401 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson78501ea2010-10-27 12:18:21 +0100402 int ret = init_ring_common(ring);
Zhenyu Wanga69ffdb2010-08-30 16:12:42 +0800403
Chris Wilsona6c45cf2010-09-17 00:32:17 +0100404 if (INTEL_INFO(dev)->gen > 3) {
Daniel Vetter6b26c862012-04-24 14:04:12 +0200405 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
Jesse Barnesb095cd02011-08-12 15:28:32 -0700406 if (IS_GEN7(dev))
407 I915_WRITE(GFX_MODE_GEN7,
Daniel Vetter6b26c862012-04-24 14:04:12 +0200408 _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
409 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800410 }
Chris Wilson78501ea2010-10-27 12:18:21 +0100411
Jesse Barnes8d315282011-10-16 10:23:31 +0200412 if (INTEL_INFO(dev)->gen >= 5) {
Chris Wilsonc6df5412010-12-15 09:56:50 +0000413 ret = init_pipe_control(ring);
414 if (ret)
415 return ret;
416 }
417
Daniel Vetter5e13a0c2012-05-08 13:39:59 +0200418 if (IS_GEN6(dev)) {
Kenneth Graunke3a69ddd2012-04-27 12:44:41 -0700419 /* From the Sandybridge PRM, volume 1 part 3, page 24:
420 * "If this bit is set, STCunit will have LRA as replacement
421 * policy. [...] This bit must be reset. LRA replacement
422 * policy is not supported."
423 */
424 I915_WRITE(CACHE_MODE_0,
Daniel Vetter5e13a0c2012-05-08 13:39:59 +0200425 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
Ben Widawsky84f9f932011-12-12 19:21:58 -0800426 }
427
Daniel Vetter6b26c862012-04-24 14:04:12 +0200428 if (INTEL_INFO(dev)->gen >= 6)
429 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
Chris Wilsonc6df5412010-12-15 09:56:50 +0000430
Ben Widawsky15b9f802012-05-25 16:56:23 -0700431 if (IS_IVYBRIDGE(dev))
432 I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
433
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800434 return ret;
435}
436
Chris Wilsonc6df5412010-12-15 09:56:50 +0000437static void render_ring_cleanup(struct intel_ring_buffer *ring)
438{
439 if (!ring->private)
440 return;
441
442 cleanup_pipe_control(ring);
443}
444
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000445static void
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700446update_mboxes(struct intel_ring_buffer *ring,
447 u32 seqno,
448 u32 mmio_offset)
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000449{
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700450 intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
451 MI_SEMAPHORE_GLOBAL_GTT |
452 MI_SEMAPHORE_REGISTER |
453 MI_SEMAPHORE_UPDATE);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000454 intel_ring_emit(ring, seqno);
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700455 intel_ring_emit(ring, mmio_offset);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000456}
457
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700458/**
459 * gen6_add_request - Update the semaphore mailbox registers
460 *
461 * @ring - ring that is adding a request
462 * @seqno - return seqno stuck into the ring
463 *
464 * Update the mailbox registers in the *other* rings with the current seqno.
465 * This acts like a signal in the canonical semaphore.
466 */
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000467static int
468gen6_add_request(struct intel_ring_buffer *ring,
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700469 u32 *seqno)
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000470{
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700471 u32 mbox1_reg;
472 u32 mbox2_reg;
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000473 int ret;
474
475 ret = intel_ring_begin(ring, 10);
476 if (ret)
477 return ret;
478
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700479 mbox1_reg = ring->signal_mbox[0];
480 mbox2_reg = ring->signal_mbox[1];
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000481
Daniel Vetter53d227f2012-01-25 16:32:49 +0100482 *seqno = i915_gem_next_request_seqno(ring);
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700483
484 update_mboxes(ring, *seqno, mbox1_reg);
485 update_mboxes(ring, *seqno, mbox2_reg);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000486 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
487 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700488 intel_ring_emit(ring, *seqno);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000489 intel_ring_emit(ring, MI_USER_INTERRUPT);
490 intel_ring_advance(ring);
491
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000492 return 0;
493}
494
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700495/**
496 * intel_ring_sync - sync the waiter to the signaller on seqno
497 *
498 * @waiter - ring that is waiting
499 * @signaller - ring which has, or will signal
500 * @seqno - seqno which the waiter will block on
501 */
502static int
Daniel Vetter686cb5f2012-04-11 22:12:52 +0200503gen6_ring_sync(struct intel_ring_buffer *waiter,
504 struct intel_ring_buffer *signaller,
505 u32 seqno)
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000506{
507 int ret;
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700508 u32 dw1 = MI_SEMAPHORE_MBOX |
509 MI_SEMAPHORE_COMPARE |
510 MI_SEMAPHORE_REGISTER;
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000511
Ben Widawsky1500f7e2012-04-11 11:18:21 -0700512 /* Throughout all of the GEM code, seqno passed implies our current
513 * seqno is >= the last seqno executed. However for hardware the
514 * comparison is strictly greater than.
515 */
516 seqno -= 1;
517
Daniel Vetter686cb5f2012-04-11 22:12:52 +0200518 WARN_ON(signaller->semaphore_register[waiter->id] ==
519 MI_SEMAPHORE_SYNC_INVALID);
520
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700521 ret = intel_ring_begin(waiter, 4);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000522 if (ret)
523 return ret;
524
Daniel Vetter686cb5f2012-04-11 22:12:52 +0200525 intel_ring_emit(waiter,
526 dw1 | signaller->semaphore_register[waiter->id]);
Ben Widawskyc8c99b02011-09-14 20:32:47 -0700527 intel_ring_emit(waiter, seqno);
528 intel_ring_emit(waiter, 0);
529 intel_ring_emit(waiter, MI_NOOP);
530 intel_ring_advance(waiter);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000531
532 return 0;
533}
534
Chris Wilsonc6df5412010-12-15 09:56:50 +0000535#define PIPE_CONTROL_FLUSH(ring__, addr__) \
536do { \
Kenneth Graunkefcbc34e2011-10-11 23:41:08 +0200537 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \
538 PIPE_CONTROL_DEPTH_STALL); \
Chris Wilsonc6df5412010-12-15 09:56:50 +0000539 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
540 intel_ring_emit(ring__, 0); \
541 intel_ring_emit(ring__, 0); \
542} while (0)
543
544static int
545pc_render_add_request(struct intel_ring_buffer *ring,
546 u32 *result)
547{
Daniel Vetter53d227f2012-01-25 16:32:49 +0100548 u32 seqno = i915_gem_next_request_seqno(ring);
Chris Wilsonc6df5412010-12-15 09:56:50 +0000549 struct pipe_control *pc = ring->private;
550 u32 scratch_addr = pc->gtt_offset + 128;
551 int ret;
552
553 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
554 * incoherent with writes to memory, i.e. completely fubar,
555 * so we need to use PIPE_NOTIFY instead.
556 *
557 * However, we also need to workaround the qword write
558 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
559 * memory before requesting an interrupt.
560 */
561 ret = intel_ring_begin(ring, 32);
562 if (ret)
563 return ret;
564
Kenneth Graunkefcbc34e2011-10-11 23:41:08 +0200565 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
Kenneth Graunke9d971b32011-10-11 23:41:09 +0200566 PIPE_CONTROL_WRITE_FLUSH |
567 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
Chris Wilsonc6df5412010-12-15 09:56:50 +0000568 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
569 intel_ring_emit(ring, seqno);
570 intel_ring_emit(ring, 0);
571 PIPE_CONTROL_FLUSH(ring, scratch_addr);
572 scratch_addr += 128; /* write to separate cachelines */
573 PIPE_CONTROL_FLUSH(ring, scratch_addr);
574 scratch_addr += 128;
575 PIPE_CONTROL_FLUSH(ring, scratch_addr);
576 scratch_addr += 128;
577 PIPE_CONTROL_FLUSH(ring, scratch_addr);
578 scratch_addr += 128;
579 PIPE_CONTROL_FLUSH(ring, scratch_addr);
580 scratch_addr += 128;
581 PIPE_CONTROL_FLUSH(ring, scratch_addr);
Chris Wilsona71d8d92012-02-15 11:25:36 +0000582
Kenneth Graunkefcbc34e2011-10-11 23:41:08 +0200583 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
Kenneth Graunke9d971b32011-10-11 23:41:09 +0200584 PIPE_CONTROL_WRITE_FLUSH |
585 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
Chris Wilsonc6df5412010-12-15 09:56:50 +0000586 PIPE_CONTROL_NOTIFY);
587 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
588 intel_ring_emit(ring, seqno);
589 intel_ring_emit(ring, 0);
590 intel_ring_advance(ring);
591
592 *result = seqno;
593 return 0;
594}
595
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800596static u32
Daniel Vetter4cd53c02012-12-14 16:01:25 +0100597gen6_ring_get_seqno(struct intel_ring_buffer *ring)
598{
599 struct drm_device *dev = ring->dev;
600
601 /* Workaround to force correct ordering between irq and seqno writes on
602 * ivb (and maybe also on snb) by reading from a CS register (like
603 * ACTHD) before reading the status page. */
Daniel Vetter1c7eaac2012-03-27 09:31:24 +0200604 if (IS_GEN6(dev) || IS_GEN7(dev))
Daniel Vetter4cd53c02012-12-14 16:01:25 +0100605 intel_ring_get_active_head(ring);
606 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
607}
608
609static u32
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000610ring_get_seqno(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800611{
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000612 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
613}
614
Chris Wilsonc6df5412010-12-15 09:56:50 +0000615static u32
616pc_render_get_seqno(struct intel_ring_buffer *ring)
617{
618 struct pipe_control *pc = ring->private;
619 return pc->cpu_page[0];
620}
621
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000622static bool
Daniel Vettere48d8632012-04-11 22:12:54 +0200623gen5_ring_get_irq(struct intel_ring_buffer *ring)
624{
625 struct drm_device *dev = ring->dev;
626 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson7338aef2012-04-24 21:48:47 +0100627 unsigned long flags;
Daniel Vettere48d8632012-04-11 22:12:54 +0200628
629 if (!dev->irq_enabled)
630 return false;
631
Chris Wilson7338aef2012-04-24 21:48:47 +0100632 spin_lock_irqsave(&dev_priv->irq_lock, flags);
Daniel Vetterf637fde2012-04-11 22:12:59 +0200633 if (ring->irq_refcount++ == 0) {
634 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
635 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
636 POSTING_READ(GTIMR);
637 }
Chris Wilson7338aef2012-04-24 21:48:47 +0100638 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
Daniel Vettere48d8632012-04-11 22:12:54 +0200639
640 return true;
641}
642
643static void
644gen5_ring_put_irq(struct intel_ring_buffer *ring)
645{
646 struct drm_device *dev = ring->dev;
647 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson7338aef2012-04-24 21:48:47 +0100648 unsigned long flags;
Daniel Vettere48d8632012-04-11 22:12:54 +0200649
Chris Wilson7338aef2012-04-24 21:48:47 +0100650 spin_lock_irqsave(&dev_priv->irq_lock, flags);
Daniel Vetterf637fde2012-04-11 22:12:59 +0200651 if (--ring->irq_refcount == 0) {
652 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
653 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
654 POSTING_READ(GTIMR);
655 }
Chris Wilson7338aef2012-04-24 21:48:47 +0100656 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
Daniel Vettere48d8632012-04-11 22:12:54 +0200657}
658
659static bool
Daniel Vettere3670312012-04-11 22:12:53 +0200660i9xx_ring_get_irq(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700661{
Chris Wilson78501ea2010-10-27 12:18:21 +0100662 struct drm_device *dev = ring->dev;
Chris Wilson01a03332011-01-04 22:22:56 +0000663 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson7338aef2012-04-24 21:48:47 +0100664 unsigned long flags;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700665
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000666 if (!dev->irq_enabled)
667 return false;
668
Chris Wilson7338aef2012-04-24 21:48:47 +0100669 spin_lock_irqsave(&dev_priv->irq_lock, flags);
Daniel Vetterf637fde2012-04-11 22:12:59 +0200670 if (ring->irq_refcount++ == 0) {
671 dev_priv->irq_mask &= ~ring->irq_enable_mask;
672 I915_WRITE(IMR, dev_priv->irq_mask);
673 POSTING_READ(IMR);
674 }
Chris Wilson7338aef2012-04-24 21:48:47 +0100675 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000676
677 return true;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700678}
679
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800680static void
Daniel Vettere3670312012-04-11 22:12:53 +0200681i9xx_ring_put_irq(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700682{
Chris Wilson78501ea2010-10-27 12:18:21 +0100683 struct drm_device *dev = ring->dev;
Chris Wilson01a03332011-01-04 22:22:56 +0000684 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson7338aef2012-04-24 21:48:47 +0100685 unsigned long flags;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700686
Chris Wilson7338aef2012-04-24 21:48:47 +0100687 spin_lock_irqsave(&dev_priv->irq_lock, flags);
Daniel Vetterf637fde2012-04-11 22:12:59 +0200688 if (--ring->irq_refcount == 0) {
689 dev_priv->irq_mask |= ring->irq_enable_mask;
690 I915_WRITE(IMR, dev_priv->irq_mask);
691 POSTING_READ(IMR);
692 }
Chris Wilson7338aef2012-04-24 21:48:47 +0100693 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700694}
695
Chris Wilsonc2798b12012-04-22 21:13:57 +0100696static bool
697i8xx_ring_get_irq(struct intel_ring_buffer *ring)
698{
699 struct drm_device *dev = ring->dev;
700 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson7338aef2012-04-24 21:48:47 +0100701 unsigned long flags;
Chris Wilsonc2798b12012-04-22 21:13:57 +0100702
703 if (!dev->irq_enabled)
704 return false;
705
Chris Wilson7338aef2012-04-24 21:48:47 +0100706 spin_lock_irqsave(&dev_priv->irq_lock, flags);
Chris Wilsonc2798b12012-04-22 21:13:57 +0100707 if (ring->irq_refcount++ == 0) {
708 dev_priv->irq_mask &= ~ring->irq_enable_mask;
709 I915_WRITE16(IMR, dev_priv->irq_mask);
710 POSTING_READ16(IMR);
711 }
Chris Wilson7338aef2012-04-24 21:48:47 +0100712 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
Chris Wilsonc2798b12012-04-22 21:13:57 +0100713
714 return true;
715}
716
717static void
718i8xx_ring_put_irq(struct intel_ring_buffer *ring)
719{
720 struct drm_device *dev = ring->dev;
721 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson7338aef2012-04-24 21:48:47 +0100722 unsigned long flags;
Chris Wilsonc2798b12012-04-22 21:13:57 +0100723
Chris Wilson7338aef2012-04-24 21:48:47 +0100724 spin_lock_irqsave(&dev_priv->irq_lock, flags);
Chris Wilsonc2798b12012-04-22 21:13:57 +0100725 if (--ring->irq_refcount == 0) {
726 dev_priv->irq_mask |= ring->irq_enable_mask;
727 I915_WRITE16(IMR, dev_priv->irq_mask);
728 POSTING_READ16(IMR);
729 }
Chris Wilson7338aef2012-04-24 21:48:47 +0100730 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
Chris Wilsonc2798b12012-04-22 21:13:57 +0100731}
732
Chris Wilson78501ea2010-10-27 12:18:21 +0100733void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800734{
Eric Anholt45930102011-05-06 17:12:35 -0700735 struct drm_device *dev = ring->dev;
Chris Wilson78501ea2010-10-27 12:18:21 +0100736 drm_i915_private_t *dev_priv = ring->dev->dev_private;
Eric Anholt45930102011-05-06 17:12:35 -0700737 u32 mmio = 0;
738
739 /* The ring status page addresses are no longer next to the rest of
740 * the ring registers as of gen7.
741 */
742 if (IS_GEN7(dev)) {
743 switch (ring->id) {
Daniel Vetter96154f22011-12-14 13:57:00 +0100744 case RCS:
Eric Anholt45930102011-05-06 17:12:35 -0700745 mmio = RENDER_HWS_PGA_GEN7;
746 break;
Daniel Vetter96154f22011-12-14 13:57:00 +0100747 case BCS:
Eric Anholt45930102011-05-06 17:12:35 -0700748 mmio = BLT_HWS_PGA_GEN7;
749 break;
Daniel Vetter96154f22011-12-14 13:57:00 +0100750 case VCS:
Eric Anholt45930102011-05-06 17:12:35 -0700751 mmio = BSD_HWS_PGA_GEN7;
752 break;
753 }
754 } else if (IS_GEN6(ring->dev)) {
755 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
756 } else {
757 mmio = RING_HWS_PGA(ring->mmio_base);
758 }
759
Chris Wilson78501ea2010-10-27 12:18:21 +0100760 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
761 POSTING_READ(mmio);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800762}
763
Chris Wilsonb72f3ac2011-01-04 17:34:02 +0000764static int
Chris Wilson78501ea2010-10-27 12:18:21 +0100765bsd_ring_flush(struct intel_ring_buffer *ring,
766 u32 invalidate_domains,
767 u32 flush_domains)
Zou Nan haid1b851f2010-05-21 09:08:57 +0800768{
Chris Wilsonb72f3ac2011-01-04 17:34:02 +0000769 int ret;
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000770
Chris Wilsonb72f3ac2011-01-04 17:34:02 +0000771 ret = intel_ring_begin(ring, 2);
772 if (ret)
773 return ret;
774
775 intel_ring_emit(ring, MI_FLUSH);
776 intel_ring_emit(ring, MI_NOOP);
777 intel_ring_advance(ring);
778 return 0;
Zou Nan haid1b851f2010-05-21 09:08:57 +0800779}
780
Chris Wilson3cce4692010-10-27 16:11:02 +0100781static int
Daniel Vetter8620a3a2012-04-11 22:12:57 +0200782i9xx_add_request(struct intel_ring_buffer *ring,
Chris Wilson3cce4692010-10-27 16:11:02 +0100783 u32 *result)
Zou Nan haid1b851f2010-05-21 09:08:57 +0800784{
785 u32 seqno;
Chris Wilson3cce4692010-10-27 16:11:02 +0100786 int ret;
787
788 ret = intel_ring_begin(ring, 4);
789 if (ret)
790 return ret;
Chris Wilson6f392d52010-08-07 11:01:22 +0100791
Daniel Vetter53d227f2012-01-25 16:32:49 +0100792 seqno = i915_gem_next_request_seqno(ring);
Chris Wilson6f392d52010-08-07 11:01:22 +0100793
Chris Wilson3cce4692010-10-27 16:11:02 +0100794 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
795 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
796 intel_ring_emit(ring, seqno);
797 intel_ring_emit(ring, MI_USER_INTERRUPT);
798 intel_ring_advance(ring);
Zou Nan haid1b851f2010-05-21 09:08:57 +0800799
Chris Wilson3cce4692010-10-27 16:11:02 +0100800 *result = seqno;
801 return 0;
Zou Nan haid1b851f2010-05-21 09:08:57 +0800802}
803
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000804static bool
Ben Widawsky25c06302012-03-29 19:11:27 -0700805gen6_ring_get_irq(struct intel_ring_buffer *ring)
Chris Wilson0f468322011-01-04 17:35:21 +0000806{
807 struct drm_device *dev = ring->dev;
Chris Wilson01a03332011-01-04 22:22:56 +0000808 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson7338aef2012-04-24 21:48:47 +0100809 unsigned long flags;
Chris Wilson0f468322011-01-04 17:35:21 +0000810
811 if (!dev->irq_enabled)
812 return false;
813
Daniel Vetter4cd53c02012-12-14 16:01:25 +0100814 /* It looks like we need to prevent the gt from suspending while waiting
815 * for an notifiy irq, otherwise irqs seem to get lost on at least the
816 * blt/bsd rings on ivb. */
Daniel Vetter99ffa162012-01-25 14:04:00 +0100817 gen6_gt_force_wake_get(dev_priv);
Daniel Vetter4cd53c02012-12-14 16:01:25 +0100818
Chris Wilson7338aef2012-04-24 21:48:47 +0100819 spin_lock_irqsave(&dev_priv->irq_lock, flags);
Chris Wilson01a03332011-01-04 22:22:56 +0000820 if (ring->irq_refcount++ == 0) {
Ben Widawsky15b9f802012-05-25 16:56:23 -0700821 if (IS_IVYBRIDGE(dev) && ring->id == RCS)
822 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask |
823 GEN6_RENDER_L3_PARITY_ERROR));
824 else
825 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
Daniel Vetterf637fde2012-04-11 22:12:59 +0200826 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
827 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
828 POSTING_READ(GTIMR);
Chris Wilson0f468322011-01-04 17:35:21 +0000829 }
Chris Wilson7338aef2012-04-24 21:48:47 +0100830 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
Chris Wilson0f468322011-01-04 17:35:21 +0000831
832 return true;
833}
834
835static void
Ben Widawsky25c06302012-03-29 19:11:27 -0700836gen6_ring_put_irq(struct intel_ring_buffer *ring)
Chris Wilson0f468322011-01-04 17:35:21 +0000837{
838 struct drm_device *dev = ring->dev;
Chris Wilson01a03332011-01-04 22:22:56 +0000839 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson7338aef2012-04-24 21:48:47 +0100840 unsigned long flags;
Chris Wilson0f468322011-01-04 17:35:21 +0000841
Chris Wilson7338aef2012-04-24 21:48:47 +0100842 spin_lock_irqsave(&dev_priv->irq_lock, flags);
Chris Wilson01a03332011-01-04 22:22:56 +0000843 if (--ring->irq_refcount == 0) {
Ben Widawsky15b9f802012-05-25 16:56:23 -0700844 if (IS_IVYBRIDGE(dev) && ring->id == RCS)
845 I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
846 else
847 I915_WRITE_IMR(ring, ~0);
Daniel Vetterf637fde2012-04-11 22:12:59 +0200848 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
849 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
850 POSTING_READ(GTIMR);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000851 }
Chris Wilson7338aef2012-04-24 21:48:47 +0100852 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
Daniel Vetter4cd53c02012-12-14 16:01:25 +0100853
Daniel Vetter99ffa162012-01-25 14:04:00 +0100854 gen6_gt_force_wake_put(dev_priv);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000855}
856
Zou Nan haid1b851f2010-05-21 09:08:57 +0800857static int
Daniel Vetterfb3256d2012-04-11 22:12:56 +0200858i965_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
Zou Nan haid1b851f2010-05-21 09:08:57 +0800859{
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100860 int ret;
Chris Wilson78501ea2010-10-27 12:18:21 +0100861
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100862 ret = intel_ring_begin(ring, 2);
863 if (ret)
864 return ret;
865
Chris Wilson78501ea2010-10-27 12:18:21 +0100866 intel_ring_emit(ring,
Chris Wilson65f56872012-04-17 16:38:12 +0100867 MI_BATCH_BUFFER_START |
868 MI_BATCH_GTT |
Chris Wilson78501ea2010-10-27 12:18:21 +0100869 MI_BATCH_NON_SECURE_I965);
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000870 intel_ring_emit(ring, offset);
Chris Wilson78501ea2010-10-27 12:18:21 +0100871 intel_ring_advance(ring);
872
Zou Nan haid1b851f2010-05-21 09:08:57 +0800873 return 0;
874}
875
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800876static int
Daniel Vetterfb3256d2012-04-11 22:12:56 +0200877i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000878 u32 offset, u32 len)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700879{
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000880 int ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700881
Daniel Vetterfb3256d2012-04-11 22:12:56 +0200882 ret = intel_ring_begin(ring, 4);
883 if (ret)
884 return ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700885
Daniel Vetterfb3256d2012-04-11 22:12:56 +0200886 intel_ring_emit(ring, MI_BATCH_BUFFER);
887 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
888 intel_ring_emit(ring, offset + len - 8);
889 intel_ring_emit(ring, 0);
890 intel_ring_advance(ring);
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100891
Daniel Vetterfb3256d2012-04-11 22:12:56 +0200892 return 0;
893}
894
895static int
896i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
897 u32 offset, u32 len)
898{
899 int ret;
900
901 ret = intel_ring_begin(ring, 2);
902 if (ret)
903 return ret;
904
Chris Wilson65f56872012-04-17 16:38:12 +0100905 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
Daniel Vetterfb3256d2012-04-11 22:12:56 +0200906 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000907 intel_ring_advance(ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700908
Eric Anholt62fdfea2010-05-21 13:26:39 -0700909 return 0;
910}
911
Chris Wilson78501ea2010-10-27 12:18:21 +0100912static void cleanup_status_page(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700913{
Chris Wilson05394f32010-11-08 19:18:58 +0000914 struct drm_i915_gem_object *obj;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700915
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800916 obj = ring->status_page.obj;
917 if (obj == NULL)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700918 return;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700919
Chris Wilson05394f32010-11-08 19:18:58 +0000920 kunmap(obj->pages[0]);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700921 i915_gem_object_unpin(obj);
Chris Wilson05394f32010-11-08 19:18:58 +0000922 drm_gem_object_unreference(&obj->base);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800923 ring->status_page.obj = NULL;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700924}
925
Chris Wilson78501ea2010-10-27 12:18:21 +0100926static int init_status_page(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700927{
Chris Wilson78501ea2010-10-27 12:18:21 +0100928 struct drm_device *dev = ring->dev;
Chris Wilson05394f32010-11-08 19:18:58 +0000929 struct drm_i915_gem_object *obj;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700930 int ret;
931
Eric Anholt62fdfea2010-05-21 13:26:39 -0700932 obj = i915_gem_alloc_object(dev, 4096);
933 if (obj == NULL) {
934 DRM_ERROR("Failed to allocate status page\n");
935 ret = -ENOMEM;
936 goto err;
937 }
Chris Wilsone4ffd172011-04-04 09:44:39 +0100938
939 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700940
Daniel Vetter75e9e912010-11-04 17:11:09 +0100941 ret = i915_gem_object_pin(obj, 4096, true);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700942 if (ret != 0) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700943 goto err_unref;
944 }
945
Chris Wilson05394f32010-11-08 19:18:58 +0000946 ring->status_page.gfx_addr = obj->gtt_offset;
947 ring->status_page.page_addr = kmap(obj->pages[0]);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800948 if (ring->status_page.page_addr == NULL) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700949 goto err_unpin;
950 }
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800951 ring->status_page.obj = obj;
952 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700953
Chris Wilson78501ea2010-10-27 12:18:21 +0100954 intel_ring_setup_status_page(ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800955 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
956 ring->name, ring->status_page.gfx_addr);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700957
958 return 0;
959
960err_unpin:
961 i915_gem_object_unpin(obj);
962err_unref:
Chris Wilson05394f32010-11-08 19:18:58 +0000963 drm_gem_object_unreference(&obj->base);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700964err:
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800965 return ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700966}
967
Ben Widawskyc43b5632012-04-16 14:07:40 -0700968static int intel_init_ring_buffer(struct drm_device *dev,
969 struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700970{
Chris Wilson05394f32010-11-08 19:18:58 +0000971 struct drm_i915_gem_object *obj;
Daniel Vetterdd2757f2012-06-07 15:55:57 +0200972 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsondd785e32010-08-07 11:01:34 +0100973 int ret;
974
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800975 ring->dev = dev;
Chris Wilson23bc5982010-09-29 16:10:57 +0100976 INIT_LIST_HEAD(&ring->active_list);
977 INIT_LIST_HEAD(&ring->request_list);
Chris Wilson64193402010-10-24 12:38:05 +0100978 INIT_LIST_HEAD(&ring->gpu_write_list);
Daniel Vetterdfc9ef22012-04-11 22:12:47 +0200979 ring->size = 32 * PAGE_SIZE;
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000980
Chris Wilsonb259f672011-03-29 13:19:09 +0100981 init_waitqueue_head(&ring->irq_queue);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700982
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800983 if (I915_NEED_GFX_HWS(dev)) {
Chris Wilson78501ea2010-10-27 12:18:21 +0100984 ret = init_status_page(ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800985 if (ret)
986 return ret;
987 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700988
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800989 obj = i915_gem_alloc_object(dev, ring->size);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700990 if (obj == NULL) {
991 DRM_ERROR("Failed to allocate ringbuffer\n");
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800992 ret = -ENOMEM;
Chris Wilsondd785e32010-08-07 11:01:34 +0100993 goto err_hws;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700994 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700995
Chris Wilson05394f32010-11-08 19:18:58 +0000996 ring->obj = obj;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800997
Daniel Vetter75e9e912010-11-04 17:11:09 +0100998 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
Chris Wilsondd785e32010-08-07 11:01:34 +0100999 if (ret)
1000 goto err_unref;
Eric Anholt62fdfea2010-05-21 13:26:39 -07001001
Daniel Vetterdd2757f2012-06-07 15:55:57 +02001002 ring->virtual_start =
1003 ioremap_wc(dev_priv->mm.gtt->gma_bus_addr + obj->gtt_offset,
1004 ring->size);
Daniel Vetter4225d0f2012-04-26 23:28:16 +02001005 if (ring->virtual_start == NULL) {
Eric Anholt62fdfea2010-05-21 13:26:39 -07001006 DRM_ERROR("Failed to map ringbuffer.\n");
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001007 ret = -EINVAL;
Chris Wilsondd785e32010-08-07 11:01:34 +01001008 goto err_unpin;
Eric Anholt62fdfea2010-05-21 13:26:39 -07001009 }
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001010
Chris Wilson78501ea2010-10-27 12:18:21 +01001011 ret = ring->init(ring);
Chris Wilsondd785e32010-08-07 11:01:34 +01001012 if (ret)
1013 goto err_unmap;
Eric Anholt62fdfea2010-05-21 13:26:39 -07001014
Chris Wilson55249ba2010-12-22 14:04:47 +00001015 /* Workaround an erratum on the i830 which causes a hang if
1016 * the TAIL pointer points to within the last 2 cachelines
1017 * of the buffer.
1018 */
1019 ring->effective_size = ring->size;
Chris Wilson27c1cbd2012-04-09 13:59:46 +01001020 if (IS_I830(ring->dev) || IS_845G(ring->dev))
Chris Wilson55249ba2010-12-22 14:04:47 +00001021 ring->effective_size -= 128;
1022
Chris Wilsonc584fe42010-10-29 18:15:52 +01001023 return 0;
Chris Wilsondd785e32010-08-07 11:01:34 +01001024
1025err_unmap:
Daniel Vetter4225d0f2012-04-26 23:28:16 +02001026 iounmap(ring->virtual_start);
Chris Wilsondd785e32010-08-07 11:01:34 +01001027err_unpin:
1028 i915_gem_object_unpin(obj);
1029err_unref:
Chris Wilson05394f32010-11-08 19:18:58 +00001030 drm_gem_object_unreference(&obj->base);
1031 ring->obj = NULL;
Chris Wilsondd785e32010-08-07 11:01:34 +01001032err_hws:
Chris Wilson78501ea2010-10-27 12:18:21 +01001033 cleanup_status_page(ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001034 return ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -07001035}
1036
Chris Wilson78501ea2010-10-27 12:18:21 +01001037void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -07001038{
Chris Wilson33626e62010-10-29 16:18:36 +01001039 struct drm_i915_private *dev_priv;
1040 int ret;
1041
Chris Wilson05394f32010-11-08 19:18:58 +00001042 if (ring->obj == NULL)
Eric Anholt62fdfea2010-05-21 13:26:39 -07001043 return;
1044
Chris Wilson33626e62010-10-29 16:18:36 +01001045 /* Disable the ring buffer. The ring must be idle at this point */
1046 dev_priv = ring->dev->dev_private;
Ben Widawsky96f298a2011-03-19 18:14:27 -07001047 ret = intel_wait_ring_idle(ring);
Chris Wilson29ee3992011-01-24 16:35:42 +00001048 if (ret)
1049 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1050 ring->name, ret);
1051
Chris Wilson33626e62010-10-29 16:18:36 +01001052 I915_WRITE_CTL(ring, 0);
1053
Daniel Vetter4225d0f2012-04-26 23:28:16 +02001054 iounmap(ring->virtual_start);
Eric Anholt62fdfea2010-05-21 13:26:39 -07001055
Chris Wilson05394f32010-11-08 19:18:58 +00001056 i915_gem_object_unpin(ring->obj);
1057 drm_gem_object_unreference(&ring->obj->base);
1058 ring->obj = NULL;
Chris Wilson78501ea2010-10-27 12:18:21 +01001059
Zou Nan hai8d192152010-11-02 16:31:01 +08001060 if (ring->cleanup)
1061 ring->cleanup(ring);
1062
Chris Wilson78501ea2010-10-27 12:18:21 +01001063 cleanup_status_page(ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -07001064}
1065
Chris Wilson78501ea2010-10-27 12:18:21 +01001066static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -07001067{
Daniel Vetter4225d0f2012-04-26 23:28:16 +02001068 uint32_t __iomem *virt;
Chris Wilson55249ba2010-12-22 14:04:47 +00001069 int rem = ring->size - ring->tail;
Eric Anholt62fdfea2010-05-21 13:26:39 -07001070
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001071 if (ring->space < rem) {
Chris Wilson78501ea2010-10-27 12:18:21 +01001072 int ret = intel_wait_ring_buffer(ring, rem);
Eric Anholt62fdfea2010-05-21 13:26:39 -07001073 if (ret)
1074 return ret;
1075 }
Eric Anholt62fdfea2010-05-21 13:26:39 -07001076
Daniel Vetter4225d0f2012-04-26 23:28:16 +02001077 virt = ring->virtual_start + ring->tail;
1078 rem /= 4;
1079 while (rem--)
1080 iowrite32(MI_NOOP, virt++);
Eric Anholt62fdfea2010-05-21 13:26:39 -07001081
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001082 ring->tail = 0;
Chris Wilsonc7dca472011-01-20 17:00:10 +00001083 ring->space = ring_space(ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -07001084
1085 return 0;
1086}
1087
Chris Wilsona71d8d92012-02-15 11:25:36 +00001088static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
1089{
1090 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1091 bool was_interruptible;
1092 int ret;
1093
1094 /* XXX As we have not yet audited all the paths to check that
1095 * they are ready for ERESTARTSYS from intel_ring_begin, do not
1096 * allow us to be interruptible by a signal.
1097 */
1098 was_interruptible = dev_priv->mm.interruptible;
1099 dev_priv->mm.interruptible = false;
1100
Ben Widawsky199b2bc2012-05-24 15:03:11 -07001101 ret = i915_wait_seqno(ring, seqno);
Chris Wilsona71d8d92012-02-15 11:25:36 +00001102
1103 dev_priv->mm.interruptible = was_interruptible;
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07001104 if (!ret)
1105 i915_gem_retire_requests_ring(ring);
Chris Wilsona71d8d92012-02-15 11:25:36 +00001106
1107 return ret;
1108}
1109
1110static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1111{
1112 struct drm_i915_gem_request *request;
1113 u32 seqno = 0;
1114 int ret;
1115
1116 i915_gem_retire_requests_ring(ring);
1117
1118 if (ring->last_retired_head != -1) {
1119 ring->head = ring->last_retired_head;
1120 ring->last_retired_head = -1;
1121 ring->space = ring_space(ring);
1122 if (ring->space >= n)
1123 return 0;
1124 }
1125
1126 list_for_each_entry(request, &ring->request_list, list) {
1127 int space;
1128
1129 if (request->tail == -1)
1130 continue;
1131
1132 space = request->tail - (ring->tail + 8);
1133 if (space < 0)
1134 space += ring->size;
1135 if (space >= n) {
1136 seqno = request->seqno;
1137 break;
1138 }
1139
1140 /* Consume this request in case we need more space than
1141 * is available and so need to prevent a race between
1142 * updating last_retired_head and direct reads of
1143 * I915_RING_HEAD. It also provides a nice sanity check.
1144 */
1145 request->tail = -1;
1146 }
1147
1148 if (seqno == 0)
1149 return -ENOSPC;
1150
1151 ret = intel_ring_wait_seqno(ring, seqno);
1152 if (ret)
1153 return ret;
1154
1155 if (WARN_ON(ring->last_retired_head == -1))
1156 return -ENOSPC;
1157
1158 ring->head = ring->last_retired_head;
1159 ring->last_retired_head = -1;
1160 ring->space = ring_space(ring);
1161 if (WARN_ON(ring->space < n))
1162 return -ENOSPC;
1163
1164 return 0;
1165}
1166
Chris Wilson78501ea2010-10-27 12:18:21 +01001167int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
Eric Anholt62fdfea2010-05-21 13:26:39 -07001168{
Chris Wilson78501ea2010-10-27 12:18:21 +01001169 struct drm_device *dev = ring->dev;
Zou Nan haicae58522010-11-09 17:17:32 +08001170 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson78501ea2010-10-27 12:18:21 +01001171 unsigned long end;
Chris Wilsona71d8d92012-02-15 11:25:36 +00001172 int ret;
Chris Wilsonc7dca472011-01-20 17:00:10 +00001173
Chris Wilsona71d8d92012-02-15 11:25:36 +00001174 ret = intel_ring_wait_request(ring, n);
1175 if (ret != -ENOSPC)
1176 return ret;
1177
Chris Wilsondb53a302011-02-03 11:57:46 +00001178 trace_i915_ring_wait_begin(ring);
Daniel Vetter63ed2cb2012-04-23 16:50:50 +02001179 /* With GEM the hangcheck timer should kick us out of the loop,
1180 * leaving it early runs the risk of corrupting GEM state (due
1181 * to running on almost untested codepaths). But on resume
1182 * timers don't work yet, so prevent a complete hang in that
1183 * case by choosing an insanely large timeout. */
1184 end = jiffies + 60 * HZ;
Daniel Vettere6bfaf82011-12-14 13:56:59 +01001185
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001186 do {
Chris Wilsonc7dca472011-01-20 17:00:10 +00001187 ring->head = I915_READ_HEAD(ring);
1188 ring->space = ring_space(ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -07001189 if (ring->space >= n) {
Chris Wilsondb53a302011-02-03 11:57:46 +00001190 trace_i915_ring_wait_end(ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -07001191 return 0;
1192 }
1193
1194 if (dev->primary->master) {
1195 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1196 if (master_priv->sarea_priv)
1197 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1198 }
Zou Nan haid1b851f2010-05-21 09:08:57 +08001199
Chris Wilsone60a0b12010-10-13 10:09:14 +01001200 msleep(1);
Chris Wilsonf4e0b292010-10-29 21:06:16 +01001201 if (atomic_read(&dev_priv->mm.wedged))
1202 return -EAGAIN;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001203 } while (!time_after(jiffies, end));
Chris Wilsondb53a302011-02-03 11:57:46 +00001204 trace_i915_ring_wait_end(ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -07001205 return -EBUSY;
1206}
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001207
Chris Wilsone1f99ce2010-10-27 12:45:26 +01001208int intel_ring_begin(struct intel_ring_buffer *ring,
1209 int num_dwords)
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001210{
Chris Wilson21dd3732011-01-26 15:55:56 +00001211 struct drm_i915_private *dev_priv = ring->dev->dev_private;
Zou Nan haibe26a102010-06-12 17:40:24 +08001212 int n = 4*num_dwords;
Chris Wilsone1f99ce2010-10-27 12:45:26 +01001213 int ret;
Chris Wilson78501ea2010-10-27 12:18:21 +01001214
Chris Wilson21dd3732011-01-26 15:55:56 +00001215 if (unlikely(atomic_read(&dev_priv->mm.wedged)))
1216 return -EIO;
1217
Chris Wilson55249ba2010-12-22 14:04:47 +00001218 if (unlikely(ring->tail + n > ring->effective_size)) {
Chris Wilsone1f99ce2010-10-27 12:45:26 +01001219 ret = intel_wrap_ring_buffer(ring);
1220 if (unlikely(ret))
1221 return ret;
1222 }
Chris Wilson78501ea2010-10-27 12:18:21 +01001223
Chris Wilsone1f99ce2010-10-27 12:45:26 +01001224 if (unlikely(ring->space < n)) {
1225 ret = intel_wait_ring_buffer(ring, n);
1226 if (unlikely(ret))
1227 return ret;
1228 }
Chris Wilsond97ed332010-08-04 15:18:13 +01001229
1230 ring->space -= n;
Chris Wilsone1f99ce2010-10-27 12:45:26 +01001231 return 0;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001232}
1233
Chris Wilson78501ea2010-10-27 12:18:21 +01001234void intel_ring_advance(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001235{
Daniel Vettere5eb3d62012-05-03 14:48:16 +02001236 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1237
Chris Wilsond97ed332010-08-04 15:18:13 +01001238 ring->tail &= ring->size - 1;
Daniel Vettere5eb3d62012-05-03 14:48:16 +02001239 if (dev_priv->stop_rings & intel_ring_flag(ring))
1240 return;
Chris Wilson78501ea2010-10-27 12:18:21 +01001241 ring->write_tail(ring, ring->tail);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001242}
1243
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001244
Chris Wilson78501ea2010-10-27 12:18:21 +01001245static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
Chris Wilson297b0c52010-10-22 17:02:41 +01001246 u32 value)
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001247{
Akshay Joshi0206e352011-08-16 15:34:10 -04001248 drm_i915_private_t *dev_priv = ring->dev->dev_private;
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001249
1250 /* Every tail move must follow the sequence below */
Akshay Joshi0206e352011-08-16 15:34:10 -04001251 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1252 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1253 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1254 I915_WRITE(GEN6_BSD_RNCID, 0x0);
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001255
Akshay Joshi0206e352011-08-16 15:34:10 -04001256 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1257 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1258 50))
1259 DRM_ERROR("timed out waiting for IDLE Indicator\n");
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001260
Akshay Joshi0206e352011-08-16 15:34:10 -04001261 I915_WRITE_TAIL(ring, value);
1262 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1263 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1264 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001265}
1266
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001267static int gen6_ring_flush(struct intel_ring_buffer *ring,
Chris Wilson71a77e02011-02-02 12:13:49 +00001268 u32 invalidate, u32 flush)
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001269{
Chris Wilson71a77e02011-02-02 12:13:49 +00001270 uint32_t cmd;
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001271 int ret;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001272
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001273 ret = intel_ring_begin(ring, 4);
1274 if (ret)
1275 return ret;
1276
Chris Wilson71a77e02011-02-02 12:13:49 +00001277 cmd = MI_FLUSH_DW;
1278 if (invalidate & I915_GEM_GPU_DOMAINS)
1279 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1280 intel_ring_emit(ring, cmd);
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001281 intel_ring_emit(ring, 0);
1282 intel_ring_emit(ring, 0);
Chris Wilson71a77e02011-02-02 12:13:49 +00001283 intel_ring_emit(ring, MI_NOOP);
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001284 intel_ring_advance(ring);
1285 return 0;
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001286}
1287
1288static int
Chris Wilson78501ea2010-10-27 12:18:21 +01001289gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
Chris Wilsonc4e7a412010-11-30 14:10:25 +00001290 u32 offset, u32 len)
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001291{
Akshay Joshi0206e352011-08-16 15:34:10 -04001292 int ret;
Chris Wilsonab6f8e32010-09-19 17:53:44 +01001293
Akshay Joshi0206e352011-08-16 15:34:10 -04001294 ret = intel_ring_begin(ring, 2);
1295 if (ret)
1296 return ret;
Chris Wilsone1f99ce2010-10-27 12:45:26 +01001297
Akshay Joshi0206e352011-08-16 15:34:10 -04001298 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
1299 /* bit0-7 is the length on GEN6+ */
1300 intel_ring_emit(ring, offset);
1301 intel_ring_advance(ring);
Chris Wilsonab6f8e32010-09-19 17:53:44 +01001302
Akshay Joshi0206e352011-08-16 15:34:10 -04001303 return 0;
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001304}
1305
Chris Wilson549f7362010-10-19 11:19:32 +01001306/* Blitter support (SandyBridge+) */
1307
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001308static int blt_ring_flush(struct intel_ring_buffer *ring,
Chris Wilson71a77e02011-02-02 12:13:49 +00001309 u32 invalidate, u32 flush)
Zou Nan hai8d192152010-11-02 16:31:01 +08001310{
Chris Wilson71a77e02011-02-02 12:13:49 +00001311 uint32_t cmd;
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001312 int ret;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001313
Daniel Vetter6a233c72011-12-14 13:57:07 +01001314 ret = intel_ring_begin(ring, 4);
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001315 if (ret)
1316 return ret;
1317
Chris Wilson71a77e02011-02-02 12:13:49 +00001318 cmd = MI_FLUSH_DW;
1319 if (invalidate & I915_GEM_DOMAIN_RENDER)
1320 cmd |= MI_INVALIDATE_TLB;
1321 intel_ring_emit(ring, cmd);
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001322 intel_ring_emit(ring, 0);
1323 intel_ring_emit(ring, 0);
Chris Wilson71a77e02011-02-02 12:13:49 +00001324 intel_ring_emit(ring, MI_NOOP);
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001325 intel_ring_advance(ring);
1326 return 0;
Zou Nan hai8d192152010-11-02 16:31:01 +08001327}
1328
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001329int intel_init_render_ring_buffer(struct drm_device *dev)
1330{
1331 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001332 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001333
Daniel Vetter59465b52012-04-11 22:12:48 +02001334 ring->name = "render ring";
1335 ring->id = RCS;
1336 ring->mmio_base = RENDER_RING_BASE;
1337
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001338 if (INTEL_INFO(dev)->gen >= 6) {
1339 ring->add_request = gen6_add_request;
Jesse Barnes8d315282011-10-16 10:23:31 +02001340 ring->flush = gen6_render_ring_flush;
Ben Widawsky25c06302012-03-29 19:11:27 -07001341 ring->irq_get = gen6_ring_get_irq;
1342 ring->irq_put = gen6_ring_put_irq;
Daniel Vetter6a848cc2012-04-11 22:12:46 +02001343 ring->irq_enable_mask = GT_USER_INTERRUPT;
Daniel Vetter4cd53c02012-12-14 16:01:25 +01001344 ring->get_seqno = gen6_ring_get_seqno;
Daniel Vetter686cb5f2012-04-11 22:12:52 +02001345 ring->sync_to = gen6_ring_sync;
Daniel Vetter59465b52012-04-11 22:12:48 +02001346 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
1347 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
1348 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
1349 ring->signal_mbox[0] = GEN6_VRSYNC;
1350 ring->signal_mbox[1] = GEN6_BRSYNC;
Chris Wilsonc6df5412010-12-15 09:56:50 +00001351 } else if (IS_GEN5(dev)) {
1352 ring->add_request = pc_render_add_request;
Chris Wilson46f0f8d2012-04-18 11:12:11 +01001353 ring->flush = gen4_render_ring_flush;
Chris Wilsonc6df5412010-12-15 09:56:50 +00001354 ring->get_seqno = pc_render_get_seqno;
Daniel Vettere48d8632012-04-11 22:12:54 +02001355 ring->irq_get = gen5_ring_get_irq;
1356 ring->irq_put = gen5_ring_put_irq;
Daniel Vettere3670312012-04-11 22:12:53 +02001357 ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY;
Daniel Vetter59465b52012-04-11 22:12:48 +02001358 } else {
Daniel Vetter8620a3a2012-04-11 22:12:57 +02001359 ring->add_request = i9xx_add_request;
Chris Wilson46f0f8d2012-04-18 11:12:11 +01001360 if (INTEL_INFO(dev)->gen < 4)
1361 ring->flush = gen2_render_ring_flush;
1362 else
1363 ring->flush = gen4_render_ring_flush;
Daniel Vetter59465b52012-04-11 22:12:48 +02001364 ring->get_seqno = ring_get_seqno;
Chris Wilsonc2798b12012-04-22 21:13:57 +01001365 if (IS_GEN2(dev)) {
1366 ring->irq_get = i8xx_ring_get_irq;
1367 ring->irq_put = i8xx_ring_put_irq;
1368 } else {
1369 ring->irq_get = i9xx_ring_get_irq;
1370 ring->irq_put = i9xx_ring_put_irq;
1371 }
Daniel Vettere3670312012-04-11 22:12:53 +02001372 ring->irq_enable_mask = I915_USER_INTERRUPT;
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001373 }
Daniel Vetter59465b52012-04-11 22:12:48 +02001374 ring->write_tail = ring_write_tail;
Daniel Vetterfb3256d2012-04-11 22:12:56 +02001375 if (INTEL_INFO(dev)->gen >= 6)
1376 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1377 else if (INTEL_INFO(dev)->gen >= 4)
1378 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1379 else if (IS_I830(dev) || IS_845G(dev))
1380 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1381 else
1382 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
Daniel Vetter59465b52012-04-11 22:12:48 +02001383 ring->init = init_render_ring;
1384 ring->cleanup = render_ring_cleanup;
1385
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001386
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001387 if (!I915_NEED_GFX_HWS(dev)) {
1388 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1389 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1390 }
1391
1392 return intel_init_ring_buffer(dev, ring);
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001393}
1394
Chris Wilsone8616b62011-01-20 09:57:11 +00001395int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1396{
1397 drm_i915_private_t *dev_priv = dev->dev_private;
1398 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1399
Daniel Vetter59465b52012-04-11 22:12:48 +02001400 ring->name = "render ring";
1401 ring->id = RCS;
1402 ring->mmio_base = RENDER_RING_BASE;
1403
Chris Wilsone8616b62011-01-20 09:57:11 +00001404 if (INTEL_INFO(dev)->gen >= 6) {
Daniel Vetterb4178f82012-04-11 22:12:51 +02001405 /* non-kms not supported on gen6+ */
1406 return -ENODEV;
Chris Wilsone8616b62011-01-20 09:57:11 +00001407 }
Daniel Vetter28f0cbf2012-04-11 22:12:58 +02001408
1409 /* Note: gem is not supported on gen5/ilk without kms (the corresponding
1410 * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
1411 * the special gen5 functions. */
1412 ring->add_request = i9xx_add_request;
Chris Wilson46f0f8d2012-04-18 11:12:11 +01001413 if (INTEL_INFO(dev)->gen < 4)
1414 ring->flush = gen2_render_ring_flush;
1415 else
1416 ring->flush = gen4_render_ring_flush;
Daniel Vetter28f0cbf2012-04-11 22:12:58 +02001417 ring->get_seqno = ring_get_seqno;
Chris Wilsonc2798b12012-04-22 21:13:57 +01001418 if (IS_GEN2(dev)) {
1419 ring->irq_get = i8xx_ring_get_irq;
1420 ring->irq_put = i8xx_ring_put_irq;
1421 } else {
1422 ring->irq_get = i9xx_ring_get_irq;
1423 ring->irq_put = i9xx_ring_put_irq;
1424 }
Daniel Vetter28f0cbf2012-04-11 22:12:58 +02001425 ring->irq_enable_mask = I915_USER_INTERRUPT;
Daniel Vetter59465b52012-04-11 22:12:48 +02001426 ring->write_tail = ring_write_tail;
Daniel Vetterfb3256d2012-04-11 22:12:56 +02001427 if (INTEL_INFO(dev)->gen >= 4)
1428 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1429 else if (IS_I830(dev) || IS_845G(dev))
1430 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1431 else
1432 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
Daniel Vetter59465b52012-04-11 22:12:48 +02001433 ring->init = init_render_ring;
1434 ring->cleanup = render_ring_cleanup;
Chris Wilsone8616b62011-01-20 09:57:11 +00001435
Keith Packardf3234702011-07-22 10:44:39 -07001436 if (!I915_NEED_GFX_HWS(dev))
1437 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1438
Chris Wilsone8616b62011-01-20 09:57:11 +00001439 ring->dev = dev;
1440 INIT_LIST_HEAD(&ring->active_list);
1441 INIT_LIST_HEAD(&ring->request_list);
1442 INIT_LIST_HEAD(&ring->gpu_write_list);
1443
1444 ring->size = size;
1445 ring->effective_size = ring->size;
1446 if (IS_I830(ring->dev))
1447 ring->effective_size -= 128;
1448
Daniel Vetter4225d0f2012-04-26 23:28:16 +02001449 ring->virtual_start = ioremap_wc(start, size);
1450 if (ring->virtual_start == NULL) {
Chris Wilsone8616b62011-01-20 09:57:11 +00001451 DRM_ERROR("can not ioremap virtual address for"
1452 " ring buffer\n");
1453 return -ENOMEM;
1454 }
1455
Chris Wilsone8616b62011-01-20 09:57:11 +00001456 return 0;
1457}
1458
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001459int intel_init_bsd_ring_buffer(struct drm_device *dev)
1460{
1461 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001462 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001463
Daniel Vetter58fa3832012-04-11 22:12:49 +02001464 ring->name = "bsd ring";
1465 ring->id = VCS;
1466
Daniel Vetter0fd2c202012-04-11 22:12:55 +02001467 ring->write_tail = ring_write_tail;
Daniel Vetter58fa3832012-04-11 22:12:49 +02001468 if (IS_GEN6(dev) || IS_GEN7(dev)) {
1469 ring->mmio_base = GEN6_BSD_RING_BASE;
Daniel Vetter0fd2c202012-04-11 22:12:55 +02001470 /* gen6 bsd needs a special wa for tail updates */
1471 if (IS_GEN6(dev))
1472 ring->write_tail = gen6_bsd_ring_write_tail;
Daniel Vetter58fa3832012-04-11 22:12:49 +02001473 ring->flush = gen6_ring_flush;
1474 ring->add_request = gen6_add_request;
1475 ring->get_seqno = gen6_ring_get_seqno;
1476 ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
1477 ring->irq_get = gen6_ring_get_irq;
1478 ring->irq_put = gen6_ring_put_irq;
1479 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
Daniel Vetter686cb5f2012-04-11 22:12:52 +02001480 ring->sync_to = gen6_ring_sync;
Daniel Vetter58fa3832012-04-11 22:12:49 +02001481 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
1482 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
1483 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
1484 ring->signal_mbox[0] = GEN6_RVSYNC;
1485 ring->signal_mbox[1] = GEN6_BVSYNC;
1486 } else {
1487 ring->mmio_base = BSD_RING_BASE;
Daniel Vetter58fa3832012-04-11 22:12:49 +02001488 ring->flush = bsd_ring_flush;
Daniel Vetter8620a3a2012-04-11 22:12:57 +02001489 ring->add_request = i9xx_add_request;
Daniel Vetter58fa3832012-04-11 22:12:49 +02001490 ring->get_seqno = ring_get_seqno;
Daniel Vettere48d8632012-04-11 22:12:54 +02001491 if (IS_GEN5(dev)) {
Daniel Vettere3670312012-04-11 22:12:53 +02001492 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
Daniel Vettere48d8632012-04-11 22:12:54 +02001493 ring->irq_get = gen5_ring_get_irq;
1494 ring->irq_put = gen5_ring_put_irq;
1495 } else {
Daniel Vettere3670312012-04-11 22:12:53 +02001496 ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
Daniel Vettere48d8632012-04-11 22:12:54 +02001497 ring->irq_get = i9xx_ring_get_irq;
1498 ring->irq_put = i9xx_ring_put_irq;
1499 }
Daniel Vetterfb3256d2012-04-11 22:12:56 +02001500 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
Daniel Vetter58fa3832012-04-11 22:12:49 +02001501 }
1502 ring->init = init_ring_common;
1503
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001504
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001505 return intel_init_ring_buffer(dev, ring);
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001506}
Chris Wilson549f7362010-10-19 11:19:32 +01001507
1508int intel_init_blt_ring_buffer(struct drm_device *dev)
1509{
1510 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001511 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
Chris Wilson549f7362010-10-19 11:19:32 +01001512
Daniel Vetter3535d9d2012-04-11 22:12:50 +02001513 ring->name = "blitter ring";
1514 ring->id = BCS;
1515
1516 ring->mmio_base = BLT_RING_BASE;
1517 ring->write_tail = ring_write_tail;
1518 ring->flush = blt_ring_flush;
1519 ring->add_request = gen6_add_request;
1520 ring->get_seqno = gen6_ring_get_seqno;
1521 ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
1522 ring->irq_get = gen6_ring_get_irq;
1523 ring->irq_put = gen6_ring_put_irq;
1524 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
Daniel Vetter686cb5f2012-04-11 22:12:52 +02001525 ring->sync_to = gen6_ring_sync;
Daniel Vetter3535d9d2012-04-11 22:12:50 +02001526 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
1527 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
1528 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
1529 ring->signal_mbox[0] = GEN6_RBSYNC;
1530 ring->signal_mbox[1] = GEN6_VBSYNC;
1531 ring->init = init_ring_common;
Chris Wilson549f7362010-10-19 11:19:32 +01001532
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001533 return intel_init_ring_buffer(dev, ring);
Chris Wilson549f7362010-10-19 11:19:32 +01001534}