blob: 145eda241dee5c01ee9f1e1ae8725cf923f2d8f0 [file] [log] [blame]
Dan Williamsbf40a682009-09-08 17:42:55 -07001/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms and conditions of the GNU General Public License,
11 * version 2, as published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
21 *
22 * The full GNU General Public License is included in this distribution in
23 * the file called "COPYING".
24 *
25 * BSD LICENSE
26 *
27 * Copyright(c) 2004-2009 Intel Corporation. All rights reserved.
28 *
29 * Redistribution and use in source and binary forms, with or without
30 * modification, are permitted provided that the following conditions are met:
31 *
32 * * Redistributions of source code must retain the above copyright
33 * notice, this list of conditions and the following disclaimer.
34 * * Redistributions in binary form must reproduce the above copyright
35 * notice, this list of conditions and the following disclaimer in
36 * the documentation and/or other materials provided with the
37 * distribution.
38 * * Neither the name of Intel Corporation nor the names of its
39 * contributors may be used to endorse or promote products derived
40 * from this software without specific prior written permission.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
43 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
45 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
46 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
47 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
48 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
49 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
50 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
51 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
52 * POSSIBILITY OF SUCH DAMAGE.
53 */
54
55/*
56 * Support routines for v3+ hardware
57 */
58
59#include <linux/pci.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090060#include <linux/gfp.h>
Dan Williamsbf40a682009-09-08 17:42:55 -070061#include <linux/dmaengine.h>
62#include <linux/dma-mapping.h>
Paul Gortmaker70c71602011-05-22 16:47:17 -040063#include <linux/prefetch.h>
Dan Williamsbf40a682009-09-08 17:42:55 -070064#include "registers.h"
65#include "hw.h"
66#include "dma.h"
67#include "dma_v2.h"
68
Dan Williamsb094ad32009-09-08 17:42:57 -070069/* ioat hardware assumes at least two sources for raid operations */
70#define src_cnt_to_sw(x) ((x) + 2)
71#define src_cnt_to_hw(x) ((x) - 2)
72
73/* provide a lookup table for setting the source address in the base or
Dan Williamsd69d235b2009-09-08 17:42:59 -070074 * extended descriptor of an xor or pq descriptor
Dan Williamsb094ad32009-09-08 17:42:57 -070075 */
Dan Williamsd0b0c8c2011-07-22 14:20:46 -070076static const u8 xor_idx_to_desc = 0xe0;
Andi Kleen9b487ce2011-06-07 15:26:33 -070077static const u8 xor_idx_to_field[] = { 1, 4, 5, 6, 7, 0, 1, 2 };
78static const u8 pq_idx_to_desc = 0xf8;
79static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 };
Dan Williamsb094ad32009-09-08 17:42:57 -070080
81static dma_addr_t xor_get_src(struct ioat_raw_descriptor *descs[2], int idx)
82{
83 struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
84
85 return raw->field[xor_idx_to_field[idx]];
86}
87
88static void xor_set_src(struct ioat_raw_descriptor *descs[2],
89 dma_addr_t addr, u32 offset, int idx)
90{
91 struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
92
93 raw->field[xor_idx_to_field[idx]] = addr + offset;
94}
95
Dan Williamsd69d235b2009-09-08 17:42:59 -070096static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx)
97{
98 struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
99
100 return raw->field[pq_idx_to_field[idx]];
101}
102
103static void pq_set_src(struct ioat_raw_descriptor *descs[2],
104 dma_addr_t addr, u32 offset, u8 coef, int idx)
105{
106 struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0];
107 struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
108
109 raw->field[pq_idx_to_field[idx]] = addr + offset;
110 pq->coef[idx] = coef;
111}
112
Dan Williamsbf40a682009-09-08 17:42:55 -0700113static void ioat3_dma_unmap(struct ioat2_dma_chan *ioat,
Dan Williamsb094ad32009-09-08 17:42:57 -0700114 struct ioat_ring_ent *desc, int idx)
Dan Williamsbf40a682009-09-08 17:42:55 -0700115{
116 struct ioat_chan_common *chan = &ioat->base;
117 struct pci_dev *pdev = chan->device->pdev;
118 size_t len = desc->len;
119 size_t offset = len - desc->hw->size;
120 struct dma_async_tx_descriptor *tx = &desc->txd;
121 enum dma_ctrl_flags flags = tx->flags;
122
123 switch (desc->hw->ctl_f.op) {
124 case IOAT_OP_COPY:
Dan Williams58c86492009-09-08 17:43:00 -0700125 if (!desc->hw->ctl_f.null) /* skip 'interrupt' ops */
126 ioat_dma_unmap(chan, flags, len, desc->hw);
Dan Williamsbf40a682009-09-08 17:42:55 -0700127 break;
128 case IOAT_OP_FILL: {
129 struct ioat_fill_descriptor *hw = desc->fill;
130
131 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
132 ioat_unmap(pdev, hw->dst_addr - offset, len,
133 PCI_DMA_FROMDEVICE, flags, 1);
134 break;
135 }
Dan Williamsb094ad32009-09-08 17:42:57 -0700136 case IOAT_OP_XOR_VAL:
137 case IOAT_OP_XOR: {
138 struct ioat_xor_descriptor *xor = desc->xor;
139 struct ioat_ring_ent *ext;
140 struct ioat_xor_ext_descriptor *xor_ex = NULL;
141 int src_cnt = src_cnt_to_sw(xor->ctl_f.src_cnt);
142 struct ioat_raw_descriptor *descs[2];
143 int i;
144
145 if (src_cnt > 5) {
146 ext = ioat2_get_ring_ent(ioat, idx + 1);
147 xor_ex = ext->xor_ex;
148 }
149
150 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
151 descs[0] = (struct ioat_raw_descriptor *) xor;
152 descs[1] = (struct ioat_raw_descriptor *) xor_ex;
153 for (i = 0; i < src_cnt; i++) {
154 dma_addr_t src = xor_get_src(descs, i);
155
156 ioat_unmap(pdev, src - offset, len,
157 PCI_DMA_TODEVICE, flags, 0);
158 }
159
160 /* dest is a source in xor validate operations */
161 if (xor->ctl_f.op == IOAT_OP_XOR_VAL) {
162 ioat_unmap(pdev, xor->dst_addr - offset, len,
163 PCI_DMA_TODEVICE, flags, 1);
164 break;
165 }
166 }
167
168 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
169 ioat_unmap(pdev, xor->dst_addr - offset, len,
170 PCI_DMA_FROMDEVICE, flags, 1);
171 break;
172 }
Dan Williamsd69d235b2009-09-08 17:42:59 -0700173 case IOAT_OP_PQ_VAL:
174 case IOAT_OP_PQ: {
175 struct ioat_pq_descriptor *pq = desc->pq;
176 struct ioat_ring_ent *ext;
177 struct ioat_pq_ext_descriptor *pq_ex = NULL;
178 int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
179 struct ioat_raw_descriptor *descs[2];
180 int i;
181
182 if (src_cnt > 3) {
183 ext = ioat2_get_ring_ent(ioat, idx + 1);
184 pq_ex = ext->pq_ex;
185 }
186
187 /* in the 'continue' case don't unmap the dests as sources */
188 if (dmaf_p_disabled_continue(flags))
189 src_cnt--;
190 else if (dmaf_continue(flags))
191 src_cnt -= 3;
192
193 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
194 descs[0] = (struct ioat_raw_descriptor *) pq;
195 descs[1] = (struct ioat_raw_descriptor *) pq_ex;
196 for (i = 0; i < src_cnt; i++) {
197 dma_addr_t src = pq_get_src(descs, i);
198
199 ioat_unmap(pdev, src - offset, len,
200 PCI_DMA_TODEVICE, flags, 0);
201 }
202
203 /* the dests are sources in pq validate operations */
204 if (pq->ctl_f.op == IOAT_OP_XOR_VAL) {
205 if (!(flags & DMA_PREP_PQ_DISABLE_P))
206 ioat_unmap(pdev, pq->p_addr - offset,
207 len, PCI_DMA_TODEVICE, flags, 0);
208 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
209 ioat_unmap(pdev, pq->q_addr - offset,
210 len, PCI_DMA_TODEVICE, flags, 0);
211 break;
212 }
213 }
214
215 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
216 if (!(flags & DMA_PREP_PQ_DISABLE_P))
217 ioat_unmap(pdev, pq->p_addr - offset, len,
218 PCI_DMA_BIDIRECTIONAL, flags, 1);
219 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
220 ioat_unmap(pdev, pq->q_addr - offset, len,
221 PCI_DMA_BIDIRECTIONAL, flags, 1);
222 }
223 break;
224 }
Dan Williamsbf40a682009-09-08 17:42:55 -0700225 default:
226 dev_err(&pdev->dev, "%s: unknown op type: %#x\n",
227 __func__, desc->hw->ctl_f.op);
228 }
229}
230
Dan Williamsb094ad32009-09-08 17:42:57 -0700231static bool desc_has_ext(struct ioat_ring_ent *desc)
232{
233 struct ioat_dma_descriptor *hw = desc->hw;
Dan Williamsbf40a682009-09-08 17:42:55 -0700234
Dan Williamsb094ad32009-09-08 17:42:57 -0700235 if (hw->ctl_f.op == IOAT_OP_XOR ||
236 hw->ctl_f.op == IOAT_OP_XOR_VAL) {
237 struct ioat_xor_descriptor *xor = desc->xor;
238
239 if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
240 return true;
Dan Williamsd69d235b2009-09-08 17:42:59 -0700241 } else if (hw->ctl_f.op == IOAT_OP_PQ ||
242 hw->ctl_f.op == IOAT_OP_PQ_VAL) {
243 struct ioat_pq_descriptor *pq = desc->pq;
244
245 if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
246 return true;
Dan Williamsb094ad32009-09-08 17:42:57 -0700247 }
248
249 return false;
250}
251
252/**
253 * __cleanup - reclaim used descriptors
254 * @ioat: channel (ring) to clean
255 *
256 * The difference from the dma_v2.c __cleanup() is that this routine
257 * handles extended descriptors and dma-unmapping raid operations.
258 */
Dan Williamsbf40a682009-09-08 17:42:55 -0700259static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
260{
261 struct ioat_chan_common *chan = &ioat->base;
262 struct ioat_ring_ent *desc;
263 bool seen_current = false;
Dan Williams074cc472010-05-01 15:22:55 -0700264 int idx = ioat->tail, i;
Dan Williamsbf40a682009-09-08 17:42:55 -0700265 u16 active;
Dan Williamsbf40a682009-09-08 17:42:55 -0700266
267 dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
268 __func__, ioat->head, ioat->tail, ioat->issued);
269
270 active = ioat2_ring_active(ioat);
271 for (i = 0; i < active && !seen_current; i++) {
272 struct dma_async_tx_descriptor *tx;
273
Dan Williams074cc472010-05-01 15:22:55 -0700274 smp_read_barrier_depends();
275 prefetch(ioat2_get_ring_ent(ioat, idx + i + 1));
276 desc = ioat2_get_ring_ent(ioat, idx + i);
Dan Williamsbf40a682009-09-08 17:42:55 -0700277 dump_desc_dbg(ioat, desc);
278 tx = &desc->txd;
279 if (tx->cookie) {
Russell King - ARM Linuxf7fbce02012-03-06 22:35:07 +0000280 dma_cookie_complete(tx);
Dan Williams074cc472010-05-01 15:22:55 -0700281 ioat3_dma_unmap(ioat, desc, idx + i);
Dan Williamsbf40a682009-09-08 17:42:55 -0700282 if (tx->callback) {
283 tx->callback(tx->callback_param);
284 tx->callback = NULL;
285 }
286 }
287
288 if (tx->phys == phys_complete)
289 seen_current = true;
Dan Williamsb094ad32009-09-08 17:42:57 -0700290
291 /* skip extended descriptors */
292 if (desc_has_ext(desc)) {
293 BUG_ON(i + 1 >= active);
294 i++;
295 }
Dan Williamsbf40a682009-09-08 17:42:55 -0700296 }
Dan Williams074cc472010-05-01 15:22:55 -0700297 smp_mb(); /* finish all descriptor reads before incrementing tail */
298 ioat->tail = idx + i;
Dan Williamsaa75db02010-03-03 21:21:10 -0700299 BUG_ON(active && !seen_current); /* no active descs have written a completion? */
Dan Williamsbf40a682009-09-08 17:42:55 -0700300 chan->last_completion = phys_complete;
Dan Williamsb9cc9862010-03-03 21:21:13 -0700301
Dan Williams074cc472010-05-01 15:22:55 -0700302 if (active - i == 0) {
Dan Williamsbf40a682009-09-08 17:42:55 -0700303 dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
304 __func__);
305 clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
306 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
307 }
Dan Williamsb9cc9862010-03-03 21:21:13 -0700308 /* 5 microsecond delay per pending descriptor */
Dan Williams074cc472010-05-01 15:22:55 -0700309 writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK),
Dan Williamsb9cc9862010-03-03 21:21:13 -0700310 chan->device->reg_base + IOAT_INTRDELAY_OFFSET);
Dan Williamsbf40a682009-09-08 17:42:55 -0700311}
312
Dan Williams074cc472010-05-01 15:22:55 -0700313static void ioat3_cleanup(struct ioat2_dma_chan *ioat)
Dan Williamsbf40a682009-09-08 17:42:55 -0700314{
315 struct ioat_chan_common *chan = &ioat->base;
316 unsigned long phys_complete;
317
Dan Williamsb9cc9862010-03-03 21:21:13 -0700318 spin_lock_bh(&chan->cleanup_lock);
Dan Williams074cc472010-05-01 15:22:55 -0700319 if (ioat_cleanup_preamble(chan, &phys_complete))
320 __cleanup(ioat, phys_complete);
Dan Williamsb9cc9862010-03-03 21:21:13 -0700321 spin_unlock_bh(&chan->cleanup_lock);
322}
323
Dan Williamsaa4d72a2010-03-03 21:21:13 -0700324static void ioat3_cleanup_event(unsigned long data)
Dan Williamsbf40a682009-09-08 17:42:55 -0700325{
Dan Williamsaa4d72a2010-03-03 21:21:13 -0700326 struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
Dan Williamsbf40a682009-09-08 17:42:55 -0700327
Dan Williams074cc472010-05-01 15:22:55 -0700328 ioat3_cleanup(ioat);
Dan Williams773d9e22010-03-03 11:47:42 -0700329 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
Dan Williamsbf40a682009-09-08 17:42:55 -0700330}
331
332static void ioat3_restart_channel(struct ioat2_dma_chan *ioat)
333{
334 struct ioat_chan_common *chan = &ioat->base;
335 unsigned long phys_complete;
Dan Williamsbf40a682009-09-08 17:42:55 -0700336
Dan Williamsb372ec22010-03-03 11:47:42 -0700337 ioat2_quiesce(chan, 0);
Dan Williamsbf40a682009-09-08 17:42:55 -0700338 if (ioat_cleanup_preamble(chan, &phys_complete))
339 __cleanup(ioat, phys_complete);
340
341 __ioat2_restart_chan(ioat);
342}
343
344static void ioat3_timer_event(unsigned long data)
345{
Dan Williamsaa4d72a2010-03-03 21:21:13 -0700346 struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
Dan Williamsbf40a682009-09-08 17:42:55 -0700347 struct ioat_chan_common *chan = &ioat->base;
348
Dan Williamsbf40a682009-09-08 17:42:55 -0700349 if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
350 unsigned long phys_complete;
351 u64 status;
352
Dan Williamsbf40a682009-09-08 17:42:55 -0700353 status = ioat_chansts(chan);
354
355 /* when halted due to errors check for channel
356 * programming errors before advancing the completion state
357 */
358 if (is_ioat_halted(status)) {
359 u32 chanerr;
360
361 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
Dan Williamsb57014d2009-11-19 17:10:07 -0700362 dev_err(to_dev(chan), "%s: Channel halted (%x)\n",
363 __func__, chanerr);
Dan Williams556ab452010-07-23 15:47:56 -0700364 if (test_bit(IOAT_RUN, &chan->state))
365 BUG_ON(is_ioat_bug(chanerr));
366 else /* we never got off the ground */
367 return;
Dan Williamsbf40a682009-09-08 17:42:55 -0700368 }
369
370 /* if we haven't made progress and we have already
371 * acknowledged a pending completion once, then be more
372 * forceful with a restart
373 */
Dan Williams074cc472010-05-01 15:22:55 -0700374 spin_lock_bh(&chan->cleanup_lock);
Dan Williamsbf40a682009-09-08 17:42:55 -0700375 if (ioat_cleanup_preamble(chan, &phys_complete))
376 __cleanup(ioat, phys_complete);
Dan Williams074cc472010-05-01 15:22:55 -0700377 else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) {
378 spin_lock_bh(&ioat->prep_lock);
Dan Williamsbf40a682009-09-08 17:42:55 -0700379 ioat3_restart_channel(ioat);
Dan Williams074cc472010-05-01 15:22:55 -0700380 spin_unlock_bh(&ioat->prep_lock);
381 } else {
Dan Williamsbf40a682009-09-08 17:42:55 -0700382 set_bit(IOAT_COMPLETION_ACK, &chan->state);
383 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
384 }
Dan Williams074cc472010-05-01 15:22:55 -0700385 spin_unlock_bh(&chan->cleanup_lock);
Dan Williamsbf40a682009-09-08 17:42:55 -0700386 } else {
387 u16 active;
388
389 /* if the ring is idle, empty, and oversized try to step
390 * down the size
391 */
Dan Williams074cc472010-05-01 15:22:55 -0700392 spin_lock_bh(&chan->cleanup_lock);
393 spin_lock_bh(&ioat->prep_lock);
Dan Williamsbf40a682009-09-08 17:42:55 -0700394 active = ioat2_ring_active(ioat);
395 if (active == 0 && ioat->alloc_order > ioat_get_alloc_order())
396 reshape_ring(ioat, ioat->alloc_order-1);
Dan Williams074cc472010-05-01 15:22:55 -0700397 spin_unlock_bh(&ioat->prep_lock);
398 spin_unlock_bh(&chan->cleanup_lock);
Dan Williamsbf40a682009-09-08 17:42:55 -0700399
400 /* keep shrinking until we get back to our minimum
401 * default size
402 */
403 if (ioat->alloc_order > ioat_get_alloc_order())
404 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
405 }
Dan Williamsbf40a682009-09-08 17:42:55 -0700406}
407
408static enum dma_status
Linus Walleij07934482010-03-26 16:50:49 -0700409ioat3_tx_status(struct dma_chan *c, dma_cookie_t cookie,
410 struct dma_tx_state *txstate)
Dan Williamsbf40a682009-09-08 17:42:55 -0700411{
412 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +0000413 enum dma_status ret;
Dan Williamsbf40a682009-09-08 17:42:55 -0700414
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +0000415 ret = dma_cookie_status(c, cookie, txstate);
416 if (ret == DMA_SUCCESS)
417 return ret;
Dan Williamsbf40a682009-09-08 17:42:55 -0700418
Dan Williams074cc472010-05-01 15:22:55 -0700419 ioat3_cleanup(ioat);
Dan Williamsbf40a682009-09-08 17:42:55 -0700420
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +0000421 return dma_cookie_status(c, cookie, txstate);
Dan Williamsbf40a682009-09-08 17:42:55 -0700422}
423
424static struct dma_async_tx_descriptor *
425ioat3_prep_memset_lock(struct dma_chan *c, dma_addr_t dest, int value,
426 size_t len, unsigned long flags)
427{
428 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
429 struct ioat_ring_ent *desc;
430 size_t total_len = len;
431 struct ioat_fill_descriptor *fill;
Dan Williamsbf40a682009-09-08 17:42:55 -0700432 u64 src_data = (0x0101010101010101ULL) * (value & 0xff);
Dan Williams074cc472010-05-01 15:22:55 -0700433 int num_descs, idx, i;
Dan Williamsbf40a682009-09-08 17:42:55 -0700434
435 num_descs = ioat2_xferlen_to_descs(ioat, len);
Dan Williams074cc472010-05-01 15:22:55 -0700436 if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs) == 0)
437 idx = ioat->head;
Dan Williamsbf40a682009-09-08 17:42:55 -0700438 else
439 return NULL;
Dan Williamscdef57d2009-09-21 09:22:29 -0700440 i = 0;
441 do {
Dan Williamsbf40a682009-09-08 17:42:55 -0700442 size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
443
444 desc = ioat2_get_ring_ent(ioat, idx + i);
445 fill = desc->fill;
446
447 fill->size = xfer_size;
448 fill->src_data = src_data;
449 fill->dst_addr = dest;
450 fill->ctl = 0;
451 fill->ctl_f.op = IOAT_OP_FILL;
452
453 len -= xfer_size;
454 dest += xfer_size;
455 dump_desc_dbg(ioat, desc);
Dan Williamscdef57d2009-09-21 09:22:29 -0700456 } while (++i < num_descs);
Dan Williamsbf40a682009-09-08 17:42:55 -0700457
458 desc->txd.flags = flags;
459 desc->len = total_len;
460 fill->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
461 fill->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
462 fill->ctl_f.compl_write = 1;
463 dump_desc_dbg(ioat, desc);
464
465 /* we leave the channel locked to ensure in order submission */
466 return &desc->txd;
467}
468
Dan Williamsb094ad32009-09-08 17:42:57 -0700469static struct dma_async_tx_descriptor *
470__ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
471 dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,
472 size_t len, unsigned long flags)
473{
474 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
475 struct ioat_ring_ent *compl_desc;
476 struct ioat_ring_ent *desc;
477 struct ioat_ring_ent *ext;
478 size_t total_len = len;
479 struct ioat_xor_descriptor *xor;
480 struct ioat_xor_ext_descriptor *xor_ex = NULL;
481 struct ioat_dma_descriptor *hw;
Dan Williams074cc472010-05-01 15:22:55 -0700482 int num_descs, with_ext, idx, i;
Dan Williamsb094ad32009-09-08 17:42:57 -0700483 u32 offset = 0;
Dan Williamsb094ad32009-09-08 17:42:57 -0700484 u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR;
485
486 BUG_ON(src_cnt < 2);
487
488 num_descs = ioat2_xferlen_to_descs(ioat, len);
489 /* we need 2x the number of descriptors to cover greater than 5
490 * sources
491 */
492 if (src_cnt > 5) {
493 with_ext = 1;
494 num_descs *= 2;
495 } else
496 with_ext = 0;
497
498 /* completion writes from the raid engine may pass completion
499 * writes from the legacy engine, so we need one extra null
500 * (legacy) descriptor to ensure all completion writes arrive in
501 * order.
502 */
Dan Williams074cc472010-05-01 15:22:55 -0700503 if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs+1) == 0)
504 idx = ioat->head;
Dan Williamsb094ad32009-09-08 17:42:57 -0700505 else
506 return NULL;
Dan Williamscdef57d2009-09-21 09:22:29 -0700507 i = 0;
508 do {
Dan Williamsb094ad32009-09-08 17:42:57 -0700509 struct ioat_raw_descriptor *descs[2];
510 size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
511 int s;
512
513 desc = ioat2_get_ring_ent(ioat, idx + i);
514 xor = desc->xor;
515
516 /* save a branch by unconditionally retrieving the
517 * extended descriptor xor_set_src() knows to not write
518 * to it in the single descriptor case
519 */
520 ext = ioat2_get_ring_ent(ioat, idx + i + 1);
521 xor_ex = ext->xor_ex;
522
523 descs[0] = (struct ioat_raw_descriptor *) xor;
524 descs[1] = (struct ioat_raw_descriptor *) xor_ex;
525 for (s = 0; s < src_cnt; s++)
526 xor_set_src(descs, src[s], offset, s);
527 xor->size = xfer_size;
528 xor->dst_addr = dest + offset;
529 xor->ctl = 0;
530 xor->ctl_f.op = op;
531 xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt);
532
533 len -= xfer_size;
534 offset += xfer_size;
535 dump_desc_dbg(ioat, desc);
Dan Williamscdef57d2009-09-21 09:22:29 -0700536 } while ((i += 1 + with_ext) < num_descs);
Dan Williamsb094ad32009-09-08 17:42:57 -0700537
538 /* last xor descriptor carries the unmap parameters and fence bit */
539 desc->txd.flags = flags;
540 desc->len = total_len;
541 if (result)
542 desc->result = result;
543 xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
544
545 /* completion descriptor carries interrupt bit */
546 compl_desc = ioat2_get_ring_ent(ioat, idx + i);
547 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
548 hw = compl_desc->hw;
549 hw->ctl = 0;
550 hw->ctl_f.null = 1;
551 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
552 hw->ctl_f.compl_write = 1;
553 hw->size = NULL_DESC_BUFFER_SIZE;
554 dump_desc_dbg(ioat, compl_desc);
555
556 /* we leave the channel locked to ensure in order submission */
Dan Williams49954c12009-11-19 17:11:03 -0700557 return &compl_desc->txd;
Dan Williamsb094ad32009-09-08 17:42:57 -0700558}
559
560static struct dma_async_tx_descriptor *
561ioat3_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
562 unsigned int src_cnt, size_t len, unsigned long flags)
563{
564 return __ioat3_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags);
565}
566
567struct dma_async_tx_descriptor *
568ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
569 unsigned int src_cnt, size_t len,
570 enum sum_check_flags *result, unsigned long flags)
571{
572 /* the cleanup routine only sets bits on validate failure, it
573 * does not clear bits on validate success... so clear it here
574 */
575 *result = 0;
576
577 return __ioat3_prep_xor_lock(chan, result, src[0], &src[1],
578 src_cnt - 1, len, flags);
579}
580
Dan Williamsd69d235b2009-09-08 17:42:59 -0700581static void
582dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext)
583{
584 struct device *dev = to_dev(&ioat->base);
585 struct ioat_pq_descriptor *pq = desc->pq;
586 struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;
587 struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex };
588 int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
589 int i;
590
591 dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
592 " sz: %#x ctl: %#x (op: %d int: %d compl: %d pq: '%s%s' src_cnt: %d)\n",
593 desc_id(desc), (unsigned long long) desc->txd.phys,
594 (unsigned long long) (pq_ex ? pq_ex->next : pq->next),
595 desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, pq->ctl_f.int_en,
596 pq->ctl_f.compl_write,
597 pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
598 pq->ctl_f.src_cnt);
599 for (i = 0; i < src_cnt; i++)
600 dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
601 (unsigned long long) pq_get_src(descs, i), pq->coef[i]);
602 dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
603 dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
604}
605
606static struct dma_async_tx_descriptor *
607__ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
608 const dma_addr_t *dst, const dma_addr_t *src,
609 unsigned int src_cnt, const unsigned char *scf,
610 size_t len, unsigned long flags)
611{
612 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
613 struct ioat_chan_common *chan = &ioat->base;
614 struct ioat_ring_ent *compl_desc;
615 struct ioat_ring_ent *desc;
616 struct ioat_ring_ent *ext;
617 size_t total_len = len;
618 struct ioat_pq_descriptor *pq;
619 struct ioat_pq_ext_descriptor *pq_ex = NULL;
620 struct ioat_dma_descriptor *hw;
621 u32 offset = 0;
Dan Williamsd69d235b2009-09-08 17:42:59 -0700622 u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ;
Dan Williams074cc472010-05-01 15:22:55 -0700623 int i, s, idx, with_ext, num_descs;
Dan Williamsd69d235b2009-09-08 17:42:59 -0700624
625 dev_dbg(to_dev(chan), "%s\n", __func__);
626 /* the engine requires at least two sources (we provide
627 * at least 1 implied source in the DMA_PREP_CONTINUE case)
628 */
629 BUG_ON(src_cnt + dmaf_continue(flags) < 2);
630
631 num_descs = ioat2_xferlen_to_descs(ioat, len);
632 /* we need 2x the number of descriptors to cover greater than 3
Dan Williamscd788092009-12-17 13:52:39 -0700633 * sources (we need 1 extra source in the q-only continuation
634 * case and 3 extra sources in the p+q continuation case.
Dan Williamsd69d235b2009-09-08 17:42:59 -0700635 */
Dan Williamscd788092009-12-17 13:52:39 -0700636 if (src_cnt + dmaf_p_disabled_continue(flags) > 3 ||
637 (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) {
Dan Williamsd69d235b2009-09-08 17:42:59 -0700638 with_ext = 1;
639 num_descs *= 2;
640 } else
641 with_ext = 0;
642
643 /* completion writes from the raid engine may pass completion
644 * writes from the legacy engine, so we need one extra null
645 * (legacy) descriptor to ensure all completion writes arrive in
646 * order.
647 */
648 if (likely(num_descs) &&
Dan Williams074cc472010-05-01 15:22:55 -0700649 ioat2_check_space_lock(ioat, num_descs+1) == 0)
650 idx = ioat->head;
Dan Williamsd69d235b2009-09-08 17:42:59 -0700651 else
652 return NULL;
Dan Williamscdef57d2009-09-21 09:22:29 -0700653 i = 0;
654 do {
Dan Williamsd69d235b2009-09-08 17:42:59 -0700655 struct ioat_raw_descriptor *descs[2];
656 size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
657
658 desc = ioat2_get_ring_ent(ioat, idx + i);
659 pq = desc->pq;
660
661 /* save a branch by unconditionally retrieving the
662 * extended descriptor pq_set_src() knows to not write
663 * to it in the single descriptor case
664 */
665 ext = ioat2_get_ring_ent(ioat, idx + i + with_ext);
666 pq_ex = ext->pq_ex;
667
668 descs[0] = (struct ioat_raw_descriptor *) pq;
669 descs[1] = (struct ioat_raw_descriptor *) pq_ex;
670
671 for (s = 0; s < src_cnt; s++)
672 pq_set_src(descs, src[s], offset, scf[s], s);
673
674 /* see the comment for dma_maxpq in include/linux/dmaengine.h */
675 if (dmaf_p_disabled_continue(flags))
676 pq_set_src(descs, dst[1], offset, 1, s++);
677 else if (dmaf_continue(flags)) {
678 pq_set_src(descs, dst[0], offset, 0, s++);
679 pq_set_src(descs, dst[1], offset, 1, s++);
680 pq_set_src(descs, dst[1], offset, 0, s++);
681 }
682 pq->size = xfer_size;
683 pq->p_addr = dst[0] + offset;
684 pq->q_addr = dst[1] + offset;
685 pq->ctl = 0;
686 pq->ctl_f.op = op;
687 pq->ctl_f.src_cnt = src_cnt_to_hw(s);
688 pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
689 pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
690
691 len -= xfer_size;
692 offset += xfer_size;
Dan Williamscdef57d2009-09-21 09:22:29 -0700693 } while ((i += 1 + with_ext) < num_descs);
Dan Williamsd69d235b2009-09-08 17:42:59 -0700694
695 /* last pq descriptor carries the unmap parameters and fence bit */
696 desc->txd.flags = flags;
697 desc->len = total_len;
698 if (result)
699 desc->result = result;
700 pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
701 dump_pq_desc_dbg(ioat, desc, ext);
702
703 /* completion descriptor carries interrupt bit */
704 compl_desc = ioat2_get_ring_ent(ioat, idx + i);
705 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
706 hw = compl_desc->hw;
707 hw->ctl = 0;
708 hw->ctl_f.null = 1;
709 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
710 hw->ctl_f.compl_write = 1;
711 hw->size = NULL_DESC_BUFFER_SIZE;
712 dump_desc_dbg(ioat, compl_desc);
713
714 /* we leave the channel locked to ensure in order submission */
Dan Williams49954c12009-11-19 17:11:03 -0700715 return &compl_desc->txd;
Dan Williamsd69d235b2009-09-08 17:42:59 -0700716}
717
718static struct dma_async_tx_descriptor *
719ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
720 unsigned int src_cnt, const unsigned char *scf, size_t len,
721 unsigned long flags)
722{
Dan Williamsde581b62009-11-19 17:08:45 -0700723 /* specify valid address for disabled result */
724 if (flags & DMA_PREP_PQ_DISABLE_P)
725 dst[0] = dst[1];
726 if (flags & DMA_PREP_PQ_DISABLE_Q)
727 dst[1] = dst[0];
728
Dan Williamsd69d235b2009-09-08 17:42:59 -0700729 /* handle the single source multiply case from the raid6
730 * recovery path
731 */
Dan Williamsde581b62009-11-19 17:08:45 -0700732 if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) {
Dan Williamsd69d235b2009-09-08 17:42:59 -0700733 dma_addr_t single_source[2];
734 unsigned char single_source_coef[2];
735
736 BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q);
737 single_source[0] = src[0];
738 single_source[1] = src[0];
739 single_source_coef[0] = scf[0];
740 single_source_coef[1] = 0;
741
742 return __ioat3_prep_pq_lock(chan, NULL, dst, single_source, 2,
743 single_source_coef, len, flags);
744 } else
745 return __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt, scf,
746 len, flags);
747}
748
749struct dma_async_tx_descriptor *
750ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
751 unsigned int src_cnt, const unsigned char *scf, size_t len,
752 enum sum_check_flags *pqres, unsigned long flags)
753{
Dan Williamsde581b62009-11-19 17:08:45 -0700754 /* specify valid address for disabled result */
755 if (flags & DMA_PREP_PQ_DISABLE_P)
756 pq[0] = pq[1];
757 if (flags & DMA_PREP_PQ_DISABLE_Q)
758 pq[1] = pq[0];
759
Dan Williamsd69d235b2009-09-08 17:42:59 -0700760 /* the cleanup routine only sets bits on validate failure, it
761 * does not clear bits on validate success... so clear it here
762 */
763 *pqres = 0;
764
765 return __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
766 flags);
767}
768
Dan Williamsae786622009-09-08 17:43:00 -0700769static struct dma_async_tx_descriptor *
770ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
771 unsigned int src_cnt, size_t len, unsigned long flags)
772{
773 unsigned char scf[src_cnt];
774 dma_addr_t pq[2];
775
776 memset(scf, 0, src_cnt);
Dan Williamsae786622009-09-08 17:43:00 -0700777 pq[0] = dst;
Dan Williamsde581b62009-11-19 17:08:45 -0700778 flags |= DMA_PREP_PQ_DISABLE_Q;
779 pq[1] = dst; /* specify valid address for disabled result */
Dan Williamsae786622009-09-08 17:43:00 -0700780
781 return __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
782 flags);
783}
784
785struct dma_async_tx_descriptor *
786ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
787 unsigned int src_cnt, size_t len,
788 enum sum_check_flags *result, unsigned long flags)
789{
790 unsigned char scf[src_cnt];
791 dma_addr_t pq[2];
792
793 /* the cleanup routine only sets bits on validate failure, it
794 * does not clear bits on validate success... so clear it here
795 */
796 *result = 0;
797
798 memset(scf, 0, src_cnt);
Dan Williamsae786622009-09-08 17:43:00 -0700799 pq[0] = src[0];
Dan Williamsde581b62009-11-19 17:08:45 -0700800 flags |= DMA_PREP_PQ_DISABLE_Q;
801 pq[1] = pq[0]; /* specify valid address for disabled result */
Dan Williamsae786622009-09-08 17:43:00 -0700802
803 return __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, scf,
804 len, flags);
805}
806
Dan Williams58c86492009-09-08 17:43:00 -0700807static struct dma_async_tx_descriptor *
808ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
809{
810 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
811 struct ioat_ring_ent *desc;
812 struct ioat_dma_descriptor *hw;
Dan Williams58c86492009-09-08 17:43:00 -0700813
Dan Williams074cc472010-05-01 15:22:55 -0700814 if (ioat2_check_space_lock(ioat, 1) == 0)
815 desc = ioat2_get_ring_ent(ioat, ioat->head);
Dan Williams58c86492009-09-08 17:43:00 -0700816 else
817 return NULL;
818
819 hw = desc->hw;
820 hw->ctl = 0;
821 hw->ctl_f.null = 1;
822 hw->ctl_f.int_en = 1;
823 hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
824 hw->ctl_f.compl_write = 1;
825 hw->size = NULL_DESC_BUFFER_SIZE;
826 hw->src_addr = 0;
827 hw->dst_addr = 0;
828
829 desc->txd.flags = flags;
830 desc->len = 1;
831
832 dump_desc_dbg(ioat, desc);
833
834 /* we leave the channel locked to ensure in order submission */
835 return &desc->txd;
836}
837
Dan Williams9de6fc72009-09-08 17:42:58 -0700838static void __devinit ioat3_dma_test_callback(void *dma_async_param)
839{
840 struct completion *cmp = dma_async_param;
841
842 complete(cmp);
843}
844
845#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
846static int __devinit ioat_xor_val_self_test(struct ioatdma_device *device)
847{
848 int i, src_idx;
849 struct page *dest;
850 struct page *xor_srcs[IOAT_NUM_SRC_TEST];
851 struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1];
852 dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1];
853 dma_addr_t dma_addr, dest_dma;
854 struct dma_async_tx_descriptor *tx;
855 struct dma_chan *dma_chan;
856 dma_cookie_t cookie;
857 u8 cmp_byte = 0;
858 u32 cmp_word;
859 u32 xor_val_result;
860 int err = 0;
861 struct completion cmp;
862 unsigned long tmo;
863 struct device *dev = &device->pdev->dev;
864 struct dma_device *dma = &device->common;
865
866 dev_dbg(dev, "%s\n", __func__);
867
868 if (!dma_has_cap(DMA_XOR, dma->cap_mask))
869 return 0;
870
871 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
872 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
873 if (!xor_srcs[src_idx]) {
874 while (src_idx--)
875 __free_page(xor_srcs[src_idx]);
876 return -ENOMEM;
877 }
878 }
879
880 dest = alloc_page(GFP_KERNEL);
881 if (!dest) {
882 while (src_idx--)
883 __free_page(xor_srcs[src_idx]);
884 return -ENOMEM;
885 }
886
887 /* Fill in src buffers */
888 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
889 u8 *ptr = page_address(xor_srcs[src_idx]);
890 for (i = 0; i < PAGE_SIZE; i++)
891 ptr[i] = (1 << src_idx);
892 }
893
894 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++)
895 cmp_byte ^= (u8) (1 << src_idx);
896
897 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
898 (cmp_byte << 8) | cmp_byte;
899
900 memset(page_address(dest), 0, PAGE_SIZE);
901
902 dma_chan = container_of(dma->channels.next, struct dma_chan,
903 device_node);
904 if (dma->device_alloc_chan_resources(dma_chan) < 1) {
905 err = -ENODEV;
906 goto out;
907 }
908
909 /* test xor */
910 dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
911 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
912 dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE,
913 DMA_TO_DEVICE);
914 tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
915 IOAT_NUM_SRC_TEST, PAGE_SIZE,
916 DMA_PREP_INTERRUPT);
917
918 if (!tx) {
919 dev_err(dev, "Self-test xor prep failed\n");
920 err = -ENODEV;
921 goto free_resources;
922 }
923
924 async_tx_ack(tx);
925 init_completion(&cmp);
926 tx->callback = ioat3_dma_test_callback;
927 tx->callback_param = &cmp;
928 cookie = tx->tx_submit(tx);
929 if (cookie < 0) {
930 dev_err(dev, "Self-test xor setup failed\n");
931 err = -ENODEV;
932 goto free_resources;
933 }
934 dma->device_issue_pending(dma_chan);
935
936 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
937
Linus Walleij07934482010-03-26 16:50:49 -0700938 if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
Dan Williams9de6fc72009-09-08 17:42:58 -0700939 dev_err(dev, "Self-test xor timed out\n");
940 err = -ENODEV;
941 goto free_resources;
942 }
943
944 dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
945 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
946 u32 *ptr = page_address(dest);
947 if (ptr[i] != cmp_word) {
948 dev_err(dev, "Self-test xor failed compare\n");
949 err = -ENODEV;
950 goto free_resources;
951 }
952 }
953 dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_TO_DEVICE);
954
955 /* skip validate if the capability is not present */
956 if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
957 goto free_resources;
958
959 /* validate the sources with the destintation page */
960 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
961 xor_val_srcs[i] = xor_srcs[i];
962 xor_val_srcs[i] = dest;
963
964 xor_val_result = 1;
965
966 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
967 dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
968 DMA_TO_DEVICE);
969 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
970 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
971 &xor_val_result, DMA_PREP_INTERRUPT);
972 if (!tx) {
973 dev_err(dev, "Self-test zero prep failed\n");
974 err = -ENODEV;
975 goto free_resources;
976 }
977
978 async_tx_ack(tx);
979 init_completion(&cmp);
980 tx->callback = ioat3_dma_test_callback;
981 tx->callback_param = &cmp;
982 cookie = tx->tx_submit(tx);
983 if (cookie < 0) {
984 dev_err(dev, "Self-test zero setup failed\n");
985 err = -ENODEV;
986 goto free_resources;
987 }
988 dma->device_issue_pending(dma_chan);
989
990 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
991
Linus Walleij07934482010-03-26 16:50:49 -0700992 if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
Dan Williams9de6fc72009-09-08 17:42:58 -0700993 dev_err(dev, "Self-test validate timed out\n");
994 err = -ENODEV;
995 goto free_resources;
996 }
997
998 if (xor_val_result != 0) {
999 dev_err(dev, "Self-test validate failed compare\n");
1000 err = -ENODEV;
1001 goto free_resources;
1002 }
1003
1004 /* skip memset if the capability is not present */
1005 if (!dma_has_cap(DMA_MEMSET, dma_chan->device->cap_mask))
1006 goto free_resources;
1007
1008 /* test memset */
1009 dma_addr = dma_map_page(dev, dest, 0,
1010 PAGE_SIZE, DMA_FROM_DEVICE);
1011 tx = dma->device_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
1012 DMA_PREP_INTERRUPT);
1013 if (!tx) {
1014 dev_err(dev, "Self-test memset prep failed\n");
1015 err = -ENODEV;
1016 goto free_resources;
1017 }
1018
1019 async_tx_ack(tx);
1020 init_completion(&cmp);
1021 tx->callback = ioat3_dma_test_callback;
1022 tx->callback_param = &cmp;
1023 cookie = tx->tx_submit(tx);
1024 if (cookie < 0) {
1025 dev_err(dev, "Self-test memset setup failed\n");
1026 err = -ENODEV;
1027 goto free_resources;
1028 }
1029 dma->device_issue_pending(dma_chan);
1030
1031 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1032
Linus Walleij07934482010-03-26 16:50:49 -07001033 if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
Dan Williams9de6fc72009-09-08 17:42:58 -07001034 dev_err(dev, "Self-test memset timed out\n");
1035 err = -ENODEV;
1036 goto free_resources;
1037 }
1038
1039 for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
1040 u32 *ptr = page_address(dest);
1041 if (ptr[i]) {
1042 dev_err(dev, "Self-test memset failed compare\n");
1043 err = -ENODEV;
1044 goto free_resources;
1045 }
1046 }
1047
1048 /* test for non-zero parity sum */
1049 xor_val_result = 0;
1050 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1051 dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
1052 DMA_TO_DEVICE);
1053 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
1054 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
1055 &xor_val_result, DMA_PREP_INTERRUPT);
1056 if (!tx) {
1057 dev_err(dev, "Self-test 2nd zero prep failed\n");
1058 err = -ENODEV;
1059 goto free_resources;
1060 }
1061
1062 async_tx_ack(tx);
1063 init_completion(&cmp);
1064 tx->callback = ioat3_dma_test_callback;
1065 tx->callback_param = &cmp;
1066 cookie = tx->tx_submit(tx);
1067 if (cookie < 0) {
1068 dev_err(dev, "Self-test 2nd zero setup failed\n");
1069 err = -ENODEV;
1070 goto free_resources;
1071 }
1072 dma->device_issue_pending(dma_chan);
1073
1074 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1075
Linus Walleij07934482010-03-26 16:50:49 -07001076 if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
Dan Williams9de6fc72009-09-08 17:42:58 -07001077 dev_err(dev, "Self-test 2nd validate timed out\n");
1078 err = -ENODEV;
1079 goto free_resources;
1080 }
1081
1082 if (xor_val_result != SUM_CHECK_P_RESULT) {
1083 dev_err(dev, "Self-test validate failed compare\n");
1084 err = -ENODEV;
1085 goto free_resources;
1086 }
1087
1088free_resources:
1089 dma->device_free_chan_resources(dma_chan);
1090out:
1091 src_idx = IOAT_NUM_SRC_TEST;
1092 while (src_idx--)
1093 __free_page(xor_srcs[src_idx]);
1094 __free_page(dest);
1095 return err;
1096}
1097
1098static int __devinit ioat3_dma_self_test(struct ioatdma_device *device)
1099{
1100 int rc = ioat_dma_self_test(device);
1101
1102 if (rc)
1103 return rc;
1104
1105 rc = ioat_xor_val_self_test(device);
1106 if (rc)
1107 return rc;
1108
1109 return 0;
1110}
1111
Dan Williamsa6d52d72009-12-19 15:36:02 -07001112static int ioat3_reset_hw(struct ioat_chan_common *chan)
1113{
1114 /* throw away whatever the channel was doing and get it
1115 * initialized, with ioat3 specific workarounds
1116 */
1117 struct ioatdma_device *device = chan->device;
1118 struct pci_dev *pdev = device->pdev;
1119 u32 chanerr;
1120 u16 dev_id;
1121 int err;
1122
1123 ioat2_quiesce(chan, msecs_to_jiffies(100));
1124
1125 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
1126 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
1127
1128 /* -= IOAT ver.3 workarounds =- */
1129 /* Write CHANERRMSK_INT with 3E07h to mask out the errors
1130 * that can cause stability issues for IOAT ver.3, and clear any
1131 * pending errors
1132 */
1133 pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
1134 err = pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr);
1135 if (err) {
1136 dev_err(&pdev->dev, "channel error register unreachable\n");
1137 return err;
1138 }
1139 pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr);
1140
1141 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1142 * (workaround for spurious config parity error after restart)
1143 */
1144 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1145 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
1146 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
1147
1148 return ioat2_reset_sync(chan, msecs_to_jiffies(200));
1149}
1150
Dan Williamsbf40a682009-09-08 17:42:55 -07001151int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
1152{
1153 struct pci_dev *pdev = device->pdev;
Dan Williams228c4f52009-11-19 17:07:10 -07001154 int dca_en = system_has_dca_enabled(pdev);
Dan Williamsbf40a682009-09-08 17:42:55 -07001155 struct dma_device *dma;
1156 struct dma_chan *c;
1157 struct ioat_chan_common *chan;
Dan Williamse3232712009-09-08 17:43:02 -07001158 bool is_raid_device = false;
Dan Williamsbf40a682009-09-08 17:42:55 -07001159 int err;
Dan Williamsbf40a682009-09-08 17:42:55 -07001160 u32 cap;
1161
1162 device->enumerate_channels = ioat2_enumerate_channels;
Dan Williamsa6d52d72009-12-19 15:36:02 -07001163 device->reset_hw = ioat3_reset_hw;
Dan Williams9de6fc72009-09-08 17:42:58 -07001164 device->self_test = ioat3_dma_self_test;
Dan Williamsbf40a682009-09-08 17:42:55 -07001165 dma = &device->common;
1166 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
1167 dma->device_issue_pending = ioat2_issue_pending;
1168 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
1169 dma->device_free_chan_resources = ioat2_free_chan_resources;
Dan Williams58c86492009-09-08 17:43:00 -07001170
1171 dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
1172 dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock;
1173
Dan Williamsbf40a682009-09-08 17:42:55 -07001174 cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET);
Dan Williams228c4f52009-11-19 17:07:10 -07001175
1176 /* dca is incompatible with raid operations */
1177 if (dca_en && (cap & (IOAT_CAP_XOR|IOAT_CAP_PQ)))
1178 cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ);
1179
Dan Williamsb094ad32009-09-08 17:42:57 -07001180 if (cap & IOAT_CAP_XOR) {
Dan Williamse3232712009-09-08 17:43:02 -07001181 is_raid_device = true;
Dan Williamsb094ad32009-09-08 17:42:57 -07001182 dma->max_xor = 8;
Dan Williams2adfc552010-05-01 15:22:56 -07001183 dma->xor_align = 6;
Dan Williamsb094ad32009-09-08 17:42:57 -07001184
1185 dma_cap_set(DMA_XOR, dma->cap_mask);
1186 dma->device_prep_dma_xor = ioat3_prep_xor;
1187
1188 dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
1189 dma->device_prep_dma_xor_val = ioat3_prep_xor_val;
1190 }
Dan Williamsd69d235b2009-09-08 17:42:59 -07001191 if (cap & IOAT_CAP_PQ) {
Dan Williamse3232712009-09-08 17:43:02 -07001192 is_raid_device = true;
Dan Williamsd69d235b2009-09-08 17:42:59 -07001193 dma_set_maxpq(dma, 8, 0);
Dan Williams2adfc552010-05-01 15:22:56 -07001194 dma->pq_align = 6;
Dan Williamsd69d235b2009-09-08 17:42:59 -07001195
1196 dma_cap_set(DMA_PQ, dma->cap_mask);
1197 dma->device_prep_dma_pq = ioat3_prep_pq;
1198
1199 dma_cap_set(DMA_PQ_VAL, dma->cap_mask);
1200 dma->device_prep_dma_pq_val = ioat3_prep_pq_val;
Dan Williamsae786622009-09-08 17:43:00 -07001201
1202 if (!(cap & IOAT_CAP_XOR)) {
1203 dma->max_xor = 8;
Dan Williams2adfc552010-05-01 15:22:56 -07001204 dma->xor_align = 6;
Dan Williamsae786622009-09-08 17:43:00 -07001205
1206 dma_cap_set(DMA_XOR, dma->cap_mask);
1207 dma->device_prep_dma_xor = ioat3_prep_pqxor;
1208
1209 dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
1210 dma->device_prep_dma_xor_val = ioat3_prep_pqxor_val;
1211 }
Dan Williamsd69d235b2009-09-08 17:42:59 -07001212 }
Dan Williamse3232712009-09-08 17:43:02 -07001213 if (is_raid_device && (cap & IOAT_CAP_FILL_BLOCK)) {
1214 dma_cap_set(DMA_MEMSET, dma->cap_mask);
1215 dma->device_prep_dma_memset = ioat3_prep_memset_lock;
1216 }
1217
1218
1219 if (is_raid_device) {
Linus Walleij07934482010-03-26 16:50:49 -07001220 dma->device_tx_status = ioat3_tx_status;
Dan Williamsaa4d72a2010-03-03 21:21:13 -07001221 device->cleanup_fn = ioat3_cleanup_event;
Dan Williamse3232712009-09-08 17:43:02 -07001222 device->timer_fn = ioat3_timer_event;
1223 } else {
Linus Walleij07934482010-03-26 16:50:49 -07001224 dma->device_tx_status = ioat_dma_tx_status;
Dan Williamsaa4d72a2010-03-03 21:21:13 -07001225 device->cleanup_fn = ioat2_cleanup_event;
Dan Williamse3232712009-09-08 17:43:02 -07001226 device->timer_fn = ioat2_timer_event;
1227 }
Dan Williamsbf40a682009-09-08 17:42:55 -07001228
Dan Williams7b3cc2b2009-11-19 17:10:37 -07001229 #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
1230 dma_cap_clear(DMA_PQ_VAL, dma->cap_mask);
1231 dma->device_prep_dma_pq_val = NULL;
1232 #endif
1233
1234 #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
1235 dma_cap_clear(DMA_XOR_VAL, dma->cap_mask);
1236 dma->device_prep_dma_xor_val = NULL;
1237 #endif
1238
Dan Williamsbf40a682009-09-08 17:42:55 -07001239 err = ioat_probe(device);
1240 if (err)
1241 return err;
1242 ioat_set_tcp_copy_break(262144);
1243
1244 list_for_each_entry(c, &dma->channels, device_node) {
1245 chan = to_chan_common(c);
1246 writel(IOAT_DMA_DCA_ANY_CPU,
1247 chan->reg_base + IOAT_DCACTRL_OFFSET);
1248 }
1249
1250 err = ioat_register(device);
1251 if (err)
1252 return err;
Dan Williams5669e312009-09-08 17:42:56 -07001253
1254 ioat_kobject_add(device, &ioat2_ktype);
1255
Dan Williamsbf40a682009-09-08 17:42:55 -07001256 if (dca)
1257 device->dca = ioat3_dca_init(pdev, device->reg_base);
1258
1259 return 0;
1260}