blob: 6c094c88424efc7260a9dfc5e0ed522784a8f848 [file] [log] [blame]
Rusty Russell48925e32009-09-24 09:59:20 -06001/* A network driver using virtio.
Rusty Russell296f96f2007-10-22 11:03:37 +10002 *
3 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19//#define DEBUG
20#include <linux/netdevice.h>
21#include <linux/etherdevice.h>
Herbert Xua9ea3fc2008-04-18 11:21:42 +080022#include <linux/ethtool.h>
Rusty Russell296f96f2007-10-22 11:03:37 +100023#include <linux/module.h>
24#include <linux/virtio.h>
25#include <linux/virtio_net.h>
26#include <linux/scatterlist.h>
Alex Williamsone918085a2009-01-25 18:06:26 -080027#include <linux/if_vlan.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090028#include <linux/slab.h>
Rusty Russell296f96f2007-10-22 11:03:37 +100029
Dor Laor6c0cd7c2007-12-16 15:19:43 +020030static int napi_weight = 128;
31module_param(napi_weight, int, 0444);
32
Rusty Russelleb939922011-12-19 14:08:01 +000033static bool csum = true, gso = true;
Rusty Russell34a48572008-02-04 23:50:02 -050034module_param(csum, bool, 0444);
35module_param(gso, bool, 0444);
36
Rusty Russell296f96f2007-10-22 11:03:37 +100037/* FIXME: MTU in config. */
Alex Williamsone918085a2009-01-25 18:06:26 -080038#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -080039#define GOOD_COPY_LEN 128
Rusty Russell296f96f2007-10-22 11:03:37 +100040
Alex Williamsonf565a7c2009-02-04 09:02:45 +000041#define VIRTNET_SEND_COMMAND_SG_MAX 2
Rick Jones66846042011-11-14 14:17:08 +000042#define VIRTNET_DRIVER_VERSION "1.0.0"
Alex Williamson2a41f712009-02-04 09:02:34 +000043
stephen hemminger3fa2a1d2011-06-15 06:36:29 +000044struct virtnet_stats {
Eric Dumazet83a27052012-06-05 22:35:24 +000045 struct u64_stats_sync tx_syncp;
46 struct u64_stats_sync rx_syncp;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +000047 u64 tx_bytes;
48 u64 tx_packets;
49
50 u64 rx_bytes;
51 u64 rx_packets;
52};
53
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -070054struct virtnet_info {
Rusty Russell296f96f2007-10-22 11:03:37 +100055 struct virtio_device *vdev;
Alex Williamson2a41f712009-02-04 09:02:34 +000056 struct virtqueue *rvq, *svq, *cvq;
Rusty Russell296f96f2007-10-22 11:03:37 +100057 struct net_device *dev;
58 struct napi_struct napi;
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -080059 unsigned int status;
Rusty Russell296f96f2007-10-22 11:03:37 +100060
61 /* Number of input buffers, and max we've ever had. */
62 unsigned int num, max;
63
Herbert Xu97402b92008-04-18 11:24:27 +080064 /* I like... big packets and I cannot lie! */
65 bool big_packets;
66
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -080067 /* Host will merge rx buffers for big packets (shake it! shake it!) */
68 bool mergeable_rx_bufs;
69
Jason Wang586d17c2012-04-11 20:43:52 +000070 /* enable config space updates */
71 bool config_enable;
72
stephen hemminger3fa2a1d2011-06-15 06:36:29 +000073 /* Active statistics */
74 struct virtnet_stats __percpu *stats;
75
Rusty Russell3161e452009-08-26 12:22:32 -070076 /* Work struct for refilling if we run low on memory. */
77 struct delayed_work refill;
78
Jason Wang586d17c2012-04-11 20:43:52 +000079 /* Work struct for config space updates */
80 struct work_struct config_work;
81
82 /* Lock for config space updates */
83 struct mutex config_lock;
84
Rusty Russellfb6813f2008-07-25 12:06:01 -050085 /* Chain pages by the private ptr. */
86 struct page *pages;
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -070087
88 /* fragments + linear part + virtio header */
89 struct scatterlist rx_sg[MAX_SKB_FRAGS + 2];
90 struct scatterlist tx_sg[MAX_SKB_FRAGS + 2];
Rusty Russell296f96f2007-10-22 11:03:37 +100091};
92
Rusty Russellb3f24692009-09-24 09:59:19 -060093struct skb_vnet_hdr {
94 union {
95 struct virtio_net_hdr hdr;
96 struct virtio_net_hdr_mrg_rxbuf mhdr;
97 };
98};
99
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000100struct padded_vnet_hdr {
101 struct virtio_net_hdr hdr;
102 /*
103 * virtio_net_hdr should be in a separated sg buffer because of a
104 * QEMU bug, and data sg buffer shares same page with this header sg.
105 * This padding makes next sg 16 byte aligned after virtio_net_hdr.
106 */
107 char padding[6];
108};
109
Rusty Russellb3f24692009-09-24 09:59:19 -0600110static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
Rusty Russell296f96f2007-10-22 11:03:37 +1000111{
Rusty Russellb3f24692009-09-24 09:59:19 -0600112 return (struct skb_vnet_hdr *)skb->cb;
Rusty Russell296f96f2007-10-22 11:03:37 +1000113}
114
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000115/*
116 * private is used to chain pages for big packets, put the whole
117 * most recent used list in the beginning for reuse
118 */
119static void give_pages(struct virtnet_info *vi, struct page *page)
Rusty Russellfb6813f2008-07-25 12:06:01 -0500120{
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000121 struct page *end;
122
123 /* Find end of list, sew whole thing into vi->pages. */
124 for (end = page; end->private; end = (struct page *)end->private);
125 end->private = (unsigned long)vi->pages;
Rusty Russellfb6813f2008-07-25 12:06:01 -0500126 vi->pages = page;
127}
128
129static struct page *get_a_page(struct virtnet_info *vi, gfp_t gfp_mask)
130{
131 struct page *p = vi->pages;
132
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000133 if (p) {
Rusty Russellfb6813f2008-07-25 12:06:01 -0500134 vi->pages = (struct page *)p->private;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000135 /* clear private here, it is used to chain pages */
136 p->private = 0;
137 } else
Rusty Russellfb6813f2008-07-25 12:06:01 -0500138 p = alloc_page(gfp_mask);
139 return p;
140}
141
Rusty Russell2cb9c6b2008-02-04 23:50:07 -0500142static void skb_xmit_done(struct virtqueue *svq)
Rusty Russell296f96f2007-10-22 11:03:37 +1000143{
Rusty Russell2cb9c6b2008-02-04 23:50:07 -0500144 struct virtnet_info *vi = svq->vdev->priv;
Rusty Russell296f96f2007-10-22 11:03:37 +1000145
Rusty Russell2cb9c6b2008-02-04 23:50:07 -0500146 /* Suppress further interrupts. */
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300147 virtqueue_disable_cb(svq);
Rusty Russell11a3a152008-05-26 17:48:13 +1000148
Rusty Russell363f1512008-06-08 20:51:55 +1000149 /* We were probably waiting for more output buffers. */
Rusty Russell296f96f2007-10-22 11:03:37 +1000150 netif_wake_queue(vi->dev);
Rusty Russell296f96f2007-10-22 11:03:37 +1000151}
152
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000153static void set_skb_frag(struct sk_buff *skb, struct page *page,
154 unsigned int offset, unsigned int *len)
155{
Krishna Kumar8a59a7b2011-10-19 22:17:27 +0000156 int size = min((unsigned)PAGE_SIZE - offset, *len);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000157 int i = skb_shinfo(skb)->nr_frags;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000158
Krishna Kumar8a59a7b2011-10-19 22:17:27 +0000159 __skb_fill_page_desc(skb, i, page, offset, size);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000160
Krishna Kumar8a59a7b2011-10-19 22:17:27 +0000161 skb->data_len += size;
162 skb->len += size;
Eric Dumazet4b727362011-10-19 23:14:46 +0000163 skb->truesize += PAGE_SIZE;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000164 skb_shinfo(skb)->nr_frags++;
Krishna Kumar8a59a7b2011-10-19 22:17:27 +0000165 *len -= size;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000166}
167
Mike Waychison34646452012-01-04 12:52:32 +0000168/* Called from bottom half context */
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000169static struct sk_buff *page_to_skb(struct virtnet_info *vi,
170 struct page *page, unsigned int len)
171{
172 struct sk_buff *skb;
173 struct skb_vnet_hdr *hdr;
174 unsigned int copy, hdr_len, offset;
175 char *p;
176
177 p = page_address(page);
178
179 /* copy small packet so we can reuse these pages for small data */
180 skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
181 if (unlikely(!skb))
182 return NULL;
183
184 hdr = skb_vnet_hdr(skb);
185
186 if (vi->mergeable_rx_bufs) {
187 hdr_len = sizeof hdr->mhdr;
188 offset = hdr_len;
189 } else {
190 hdr_len = sizeof hdr->hdr;
191 offset = sizeof(struct padded_vnet_hdr);
192 }
193
194 memcpy(hdr, p, hdr_len);
195
196 len -= hdr_len;
197 p += offset;
198
199 copy = len;
200 if (copy > skb_tailroom(skb))
201 copy = skb_tailroom(skb);
202 memcpy(skb_put(skb, copy), p, copy);
203
204 len -= copy;
205 offset += copy;
206
Sasha Levine878d782011-09-28 04:40:54 +0000207 /*
208 * Verify that we can indeed put this data into a skb.
209 * This is here to handle cases when the device erroneously
210 * tries to receive more than is possible. This is usually
211 * the case of a broken device.
212 */
213 if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
214 if (net_ratelimit())
215 pr_debug("%s: too much data\n", skb->dev->name);
216 dev_kfree_skb(skb);
217 return NULL;
218 }
219
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000220 while (len) {
221 set_skb_frag(skb, page, offset, &len);
222 page = (struct page *)page->private;
223 offset = 0;
224 }
225
226 if (page)
227 give_pages(vi, page);
228
229 return skb;
230}
231
232static int receive_mergeable(struct virtnet_info *vi, struct sk_buff *skb)
233{
234 struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
235 struct page *page;
236 int num_buf, i, len;
237
238 num_buf = hdr->mhdr.num_buffers;
239 while (--num_buf) {
240 i = skb_shinfo(skb)->nr_frags;
241 if (i >= MAX_SKB_FRAGS) {
242 pr_debug("%s: packet too long\n", skb->dev->name);
243 skb->dev->stats.rx_length_errors++;
244 return -EINVAL;
245 }
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300246 page = virtqueue_get_buf(vi->rvq, &len);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000247 if (!page) {
248 pr_debug("%s: rx error: %d buffers missing\n",
249 skb->dev->name, hdr->mhdr.num_buffers);
250 skb->dev->stats.rx_length_errors++;
251 return -EINVAL;
252 }
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000253
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000254 if (len > PAGE_SIZE)
255 len = PAGE_SIZE;
256
257 set_skb_frag(skb, page, 0, &len);
258
259 --vi->num;
260 }
261 return 0;
262}
263
264static void receive_buf(struct net_device *dev, void *buf, unsigned int len)
Rusty Russell296f96f2007-10-22 11:03:37 +1000265{
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800266 struct virtnet_info *vi = netdev_priv(dev);
Eric Dumazet58472a72012-02-13 06:53:41 +0000267 struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000268 struct sk_buff *skb;
269 struct page *page;
270 struct skb_vnet_hdr *hdr;
Rusty Russell296f96f2007-10-22 11:03:37 +1000271
272 if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
273 pr_debug("%s: short packet %i\n", dev->name, len);
274 dev->stats.rx_length_errors++;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000275 if (vi->mergeable_rx_bufs || vi->big_packets)
276 give_pages(vi, buf);
277 else
278 dev_kfree_skb(buf);
279 return;
Rusty Russell296f96f2007-10-22 11:03:37 +1000280 }
Rusty Russell296f96f2007-10-22 11:03:37 +1000281
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000282 if (!vi->mergeable_rx_bufs && !vi->big_packets) {
283 skb = buf;
284 len -= sizeof(struct virtio_net_hdr);
285 skb_trim(skb, len);
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800286 } else {
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000287 page = buf;
288 skb = page_to_skb(vi, page, len);
289 if (unlikely(!skb)) {
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800290 dev->stats.rx_dropped++;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000291 give_pages(vi, page);
292 return;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800293 }
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000294 if (vi->mergeable_rx_bufs)
295 if (receive_mergeable(vi, skb)) {
296 dev_kfree_skb(skb);
297 return;
298 }
Herbert Xu97402b92008-04-18 11:24:27 +0800299 }
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800300
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000301 hdr = skb_vnet_hdr(skb);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000302
Eric Dumazet83a27052012-06-05 22:35:24 +0000303 u64_stats_update_begin(&stats->rx_syncp);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000304 stats->rx_bytes += skb->len;
305 stats->rx_packets++;
Eric Dumazet83a27052012-06-05 22:35:24 +0000306 u64_stats_update_end(&stats->rx_syncp);
Rusty Russell296f96f2007-10-22 11:03:37 +1000307
Rusty Russellb3f24692009-09-24 09:59:19 -0600308 if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
Rusty Russell296f96f2007-10-22 11:03:37 +1000309 pr_debug("Needs csum!\n");
Rusty Russellb3f24692009-09-24 09:59:19 -0600310 if (!skb_partial_csum_set(skb,
311 hdr->hdr.csum_start,
312 hdr->hdr.csum_offset))
Rusty Russell296f96f2007-10-22 11:03:37 +1000313 goto frame_err;
Jason Wang10a8d942011-06-10 00:56:17 +0000314 } else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
315 skb->ip_summed = CHECKSUM_UNNECESSARY;
Rusty Russell296f96f2007-10-22 11:03:37 +1000316 }
317
Mark McLoughlin23cde762008-06-08 20:49:00 +1000318 skb->protocol = eth_type_trans(skb, dev);
319 pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
320 ntohs(skb->protocol), skb->len, skb->pkt_type);
321
Rusty Russellb3f24692009-09-24 09:59:19 -0600322 if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
Rusty Russell296f96f2007-10-22 11:03:37 +1000323 pr_debug("GSO!\n");
Rusty Russellb3f24692009-09-24 09:59:19 -0600324 switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
Rusty Russell296f96f2007-10-22 11:03:37 +1000325 case VIRTIO_NET_HDR_GSO_TCPV4:
326 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
327 break;
Rusty Russell296f96f2007-10-22 11:03:37 +1000328 case VIRTIO_NET_HDR_GSO_UDP:
329 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
330 break;
331 case VIRTIO_NET_HDR_GSO_TCPV6:
332 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
333 break;
334 default:
335 if (net_ratelimit())
336 printk(KERN_WARNING "%s: bad gso type %u.\n",
Rusty Russellb3f24692009-09-24 09:59:19 -0600337 dev->name, hdr->hdr.gso_type);
Rusty Russell296f96f2007-10-22 11:03:37 +1000338 goto frame_err;
339 }
340
Rusty Russellb3f24692009-09-24 09:59:19 -0600341 if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
Rusty Russell34a48572008-02-04 23:50:02 -0500342 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
343
Rusty Russellb3f24692009-09-24 09:59:19 -0600344 skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
Rusty Russell296f96f2007-10-22 11:03:37 +1000345 if (skb_shinfo(skb)->gso_size == 0) {
346 if (net_ratelimit())
347 printk(KERN_WARNING "%s: zero gso size.\n",
348 dev->name);
349 goto frame_err;
350 }
351
352 /* Header must be checked, and gso_segs computed. */
353 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
354 skb_shinfo(skb)->gso_segs = 0;
355 }
356
357 netif_receive_skb(skb);
358 return;
359
360frame_err:
361 dev->stats.rx_frame_errors++;
Rusty Russell296f96f2007-10-22 11:03:37 +1000362 dev_kfree_skb(skb);
363}
364
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000365static int add_recvbuf_small(struct virtnet_info *vi, gfp_t gfp)
Rusty Russell296f96f2007-10-22 11:03:37 +1000366{
367 struct sk_buff *skb;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000368 struct skb_vnet_hdr *hdr;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000369 int err;
Rusty Russell296f96f2007-10-22 11:03:37 +1000370
Mike Waychison34646452012-01-04 12:52:32 +0000371 skb = __netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000372 if (unlikely(!skb))
373 return -ENOMEM;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800374
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000375 skb_put(skb, MAX_PACKET_LEN);
376
377 hdr = skb_vnet_hdr(skb);
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -0700378 sg_set_buf(vi->rx_sg, &hdr->hdr, sizeof hdr->hdr);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000379
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -0700380 skb_to_sgvec(skb, vi->rx_sg + 1, 0, skb->len);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000381
Rusty Russellf96fde42012-01-12 15:44:42 +1030382 err = virtqueue_add_buf(vi->rvq, vi->rx_sg, 0, 2, skb, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000383 if (err < 0)
384 dev_kfree_skb(skb);
385
386 return err;
387}
388
389static int add_recvbuf_big(struct virtnet_info *vi, gfp_t gfp)
390{
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000391 struct page *first, *list = NULL;
392 char *p;
393 int i, err, offset;
394
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -0700395 /* page in vi->rx_sg[MAX_SKB_FRAGS + 1] is list tail */
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000396 for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
397 first = get_a_page(vi, gfp);
398 if (!first) {
399 if (list)
400 give_pages(vi, list);
401 return -ENOMEM;
Rusty Russell3161e452009-08-26 12:22:32 -0700402 }
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -0700403 sg_set_buf(&vi->rx_sg[i], page_address(first), PAGE_SIZE);
Rusty Russell296f96f2007-10-22 11:03:37 +1000404
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000405 /* chain new page in list head to match sg */
406 first->private = (unsigned long)list;
407 list = first;
408 }
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800409
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000410 first = get_a_page(vi, gfp);
411 if (!first) {
412 give_pages(vi, list);
413 return -ENOMEM;
414 }
415 p = page_address(first);
Herbert Xu97402b92008-04-18 11:24:27 +0800416
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -0700417 /* vi->rx_sg[0], vi->rx_sg[1] share the same page */
418 /* a separated vi->rx_sg[0] for virtio_net_hdr only due to QEMU bug */
419 sg_set_buf(&vi->rx_sg[0], p, sizeof(struct virtio_net_hdr));
Herbert Xu97402b92008-04-18 11:24:27 +0800420
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -0700421 /* vi->rx_sg[1] for data packet, from offset */
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000422 offset = sizeof(struct padded_vnet_hdr);
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -0700423 sg_set_buf(&vi->rx_sg[1], p + offset, PAGE_SIZE - offset);
Herbert Xu97402b92008-04-18 11:24:27 +0800424
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000425 /* chain first in list head */
426 first->private = (unsigned long)list;
Rusty Russellf96fde42012-01-12 15:44:42 +1030427 err = virtqueue_add_buf(vi->rvq, vi->rx_sg, 0, MAX_SKB_FRAGS + 2,
428 first, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000429 if (err < 0)
430 give_pages(vi, first);
Herbert Xu97402b92008-04-18 11:24:27 +0800431
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000432 return err;
433}
Herbert Xu97402b92008-04-18 11:24:27 +0800434
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000435static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp)
436{
437 struct page *page;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000438 int err;
Rusty Russell296f96f2007-10-22 11:03:37 +1000439
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000440 page = get_a_page(vi, gfp);
441 if (!page)
442 return -ENOMEM;
443
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -0700444 sg_init_one(vi->rx_sg, page_address(page), PAGE_SIZE);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000445
Rusty Russellf96fde42012-01-12 15:44:42 +1030446 err = virtqueue_add_buf(vi->rvq, vi->rx_sg, 0, 1, page, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000447 if (err < 0)
448 give_pages(vi, page);
449
450 return err;
Rusty Russell296f96f2007-10-22 11:03:37 +1000451}
452
Rusty Russellb2baed62011-12-29 00:42:38 +0000453/*
454 * Returns false if we couldn't fill entirely (OOM).
455 *
456 * Normally run in the receive path, but can also be run from ndo_open
457 * before we're receiving packets, or from refill_work which is
458 * careful to disable receiving (using napi_disable).
459 */
Rusty Russell3161e452009-08-26 12:22:32 -0700460static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp)
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800461{
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800462 int err;
Michael S. Tsirkin1788f4952010-07-02 16:32:55 +0000463 bool oom;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800464
Amit Shah0aea51c2009-08-26 14:58:28 +0530465 do {
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000466 if (vi->mergeable_rx_bufs)
467 err = add_recvbuf_mergeable(vi, gfp);
468 else if (vi->big_packets)
469 err = add_recvbuf_big(vi, gfp);
470 else
471 err = add_recvbuf_small(vi, gfp);
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800472
Michael S. Tsirkin1788f4952010-07-02 16:32:55 +0000473 oom = err == -ENOMEM;
474 if (err < 0)
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800475 break;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000476 ++vi->num;
Amit Shah0aea51c2009-08-26 14:58:28 +0530477 } while (err > 0);
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800478 if (unlikely(vi->num > vi->max))
479 vi->max = vi->num;
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300480 virtqueue_kick(vi->rvq);
Rusty Russell3161e452009-08-26 12:22:32 -0700481 return !oom;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800482}
483
Rusty Russell18445c42008-02-04 23:49:57 -0500484static void skb_recv_done(struct virtqueue *rvq)
Rusty Russell296f96f2007-10-22 11:03:37 +1000485{
486 struct virtnet_info *vi = rvq->vdev->priv;
Rusty Russell18445c42008-02-04 23:49:57 -0500487 /* Schedule NAPI, Suppress further interrupts if successful. */
Ben Hutchings288379f2009-01-19 16:43:59 -0800488 if (napi_schedule_prep(&vi->napi)) {
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300489 virtqueue_disable_cb(rvq);
Ben Hutchings288379f2009-01-19 16:43:59 -0800490 __napi_schedule(&vi->napi);
Rusty Russell18445c42008-02-04 23:49:57 -0500491 }
Rusty Russell296f96f2007-10-22 11:03:37 +1000492}
493
Bruce Rogers3e9d08e2011-02-10 11:03:31 -0800494static void virtnet_napi_enable(struct virtnet_info *vi)
495{
496 napi_enable(&vi->napi);
497
498 /* If all buffers were filled by other side before we napi_enabled, we
499 * won't get another interrupt, so process any outstanding packets
500 * now. virtnet_poll wants re-enable the queue, so we disable here.
501 * We synchronize against interrupts via NAPI_STATE_SCHED */
502 if (napi_schedule_prep(&vi->napi)) {
503 virtqueue_disable_cb(vi->rvq);
Michael S. Tsirkinec13ee82012-05-16 10:57:12 +0300504 local_bh_disable();
Bruce Rogers3e9d08e2011-02-10 11:03:31 -0800505 __napi_schedule(&vi->napi);
Michael S. Tsirkinec13ee82012-05-16 10:57:12 +0300506 local_bh_enable();
Bruce Rogers3e9d08e2011-02-10 11:03:31 -0800507 }
508}
509
Rusty Russell3161e452009-08-26 12:22:32 -0700510static void refill_work(struct work_struct *work)
511{
512 struct virtnet_info *vi;
513 bool still_empty;
514
515 vi = container_of(work, struct virtnet_info, refill.work);
516 napi_disable(&vi->napi);
Herbert Xu39d32152010-01-25 15:51:01 -0800517 still_empty = !try_fill_recv(vi, GFP_KERNEL);
Bruce Rogers3e9d08e2011-02-10 11:03:31 -0800518 virtnet_napi_enable(vi);
Rusty Russell3161e452009-08-26 12:22:32 -0700519
520 /* In theory, this can happen: if we don't get any buffers in
521 * we will *never* try to fill again. */
522 if (still_empty)
Tejun Heo3b07e9c2012-08-20 14:51:24 -0700523 schedule_delayed_work(&vi->refill, HZ/2);
Rusty Russell3161e452009-08-26 12:22:32 -0700524}
525
Rusty Russell296f96f2007-10-22 11:03:37 +1000526static int virtnet_poll(struct napi_struct *napi, int budget)
527{
528 struct virtnet_info *vi = container_of(napi, struct virtnet_info, napi);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000529 void *buf;
Rusty Russell296f96f2007-10-22 11:03:37 +1000530 unsigned int len, received = 0;
531
532again:
533 while (received < budget &&
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300534 (buf = virtqueue_get_buf(vi->rvq, &len)) != NULL) {
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000535 receive_buf(vi->dev, buf, len);
536 --vi->num;
Rusty Russell296f96f2007-10-22 11:03:37 +1000537 received++;
538 }
539
Rusty Russell3161e452009-08-26 12:22:32 -0700540 if (vi->num < vi->max / 2) {
541 if (!try_fill_recv(vi, GFP_ATOMIC))
Tejun Heo3b07e9c2012-08-20 14:51:24 -0700542 schedule_delayed_work(&vi->refill, 0);
Rusty Russell3161e452009-08-26 12:22:32 -0700543 }
Rusty Russell296f96f2007-10-22 11:03:37 +1000544
Rusty Russell8329d982007-11-19 11:20:43 -0500545 /* Out of packets? */
546 if (received < budget) {
Ben Hutchings288379f2009-01-19 16:43:59 -0800547 napi_complete(napi);
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300548 if (unlikely(!virtqueue_enable_cb(vi->rvq)) &&
Joe Perches8e95a202009-12-03 07:58:21 +0000549 napi_schedule_prep(napi)) {
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300550 virtqueue_disable_cb(vi->rvq);
Ben Hutchings288379f2009-01-19 16:43:59 -0800551 __napi_schedule(napi);
Rusty Russell296f96f2007-10-22 11:03:37 +1000552 goto again;
Christian Borntraeger4265f162008-03-14 14:17:05 +0100553 }
Rusty Russell296f96f2007-10-22 11:03:37 +1000554 }
555
556 return received;
557}
558
Michael S. Tsirkin6ee57bc2012-10-16 23:56:14 +1030559static void free_old_xmit_skbs(struct virtnet_info *vi)
Rusty Russell296f96f2007-10-22 11:03:37 +1000560{
561 struct sk_buff *skb;
Michael S. Tsirkin6ee57bc2012-10-16 23:56:14 +1030562 unsigned int len;
Eric Dumazet58472a72012-02-13 06:53:41 +0000563 struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
Rusty Russell296f96f2007-10-22 11:03:37 +1000564
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300565 while ((skb = virtqueue_get_buf(vi->svq, &len)) != NULL) {
Rusty Russell296f96f2007-10-22 11:03:37 +1000566 pr_debug("Sent skb %p\n", skb);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000567
Eric Dumazet83a27052012-06-05 22:35:24 +0000568 u64_stats_update_begin(&stats->tx_syncp);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000569 stats->tx_bytes += skb->len;
570 stats->tx_packets++;
Eric Dumazet83a27052012-06-05 22:35:24 +0000571 u64_stats_update_end(&stats->tx_syncp);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000572
Eric Dumazeted79bab2009-10-14 14:36:43 +0000573 dev_kfree_skb_any(skb);
Rusty Russell296f96f2007-10-22 11:03:37 +1000574 }
575}
576
Rusty Russell99ffc692008-05-02 21:50:46 -0500577static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb)
Rusty Russell296f96f2007-10-22 11:03:37 +1000578{
Rusty Russellb3f24692009-09-24 09:59:19 -0600579 struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
Rusty Russell296f96f2007-10-22 11:03:37 +1000580 const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
Michael S. Tsirkin7bedc7d2012-10-16 23:56:14 +1030581 unsigned num_sg;
Rusty Russell296f96f2007-10-22 11:03:37 +1000582
Johannes Berge1749612008-10-27 15:59:26 -0700583 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
Rusty Russell296f96f2007-10-22 11:03:37 +1000584
Rusty Russell296f96f2007-10-22 11:03:37 +1000585 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Rusty Russellb3f24692009-09-24 09:59:19 -0600586 hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
Michał Mirosław55508d62010-12-14 15:24:08 +0000587 hdr->hdr.csum_start = skb_checksum_start_offset(skb);
Rusty Russellb3f24692009-09-24 09:59:19 -0600588 hdr->hdr.csum_offset = skb->csum_offset;
Rusty Russell296f96f2007-10-22 11:03:37 +1000589 } else {
Rusty Russellb3f24692009-09-24 09:59:19 -0600590 hdr->hdr.flags = 0;
591 hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
Rusty Russell296f96f2007-10-22 11:03:37 +1000592 }
593
594 if (skb_is_gso(skb)) {
Rusty Russellb3f24692009-09-24 09:59:19 -0600595 hdr->hdr.hdr_len = skb_headlen(skb);
596 hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
Rusty Russell34a48572008-02-04 23:50:02 -0500597 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
Rusty Russellb3f24692009-09-24 09:59:19 -0600598 hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
Rusty Russell296f96f2007-10-22 11:03:37 +1000599 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
Rusty Russellb3f24692009-09-24 09:59:19 -0600600 hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
Rusty Russell296f96f2007-10-22 11:03:37 +1000601 else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
Rusty Russellb3f24692009-09-24 09:59:19 -0600602 hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
Rusty Russell296f96f2007-10-22 11:03:37 +1000603 else
604 BUG();
Rusty Russell34a48572008-02-04 23:50:02 -0500605 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
Rusty Russellb3f24692009-09-24 09:59:19 -0600606 hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
Rusty Russell296f96f2007-10-22 11:03:37 +1000607 } else {
Rusty Russellb3f24692009-09-24 09:59:19 -0600608 hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
609 hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
Rusty Russell296f96f2007-10-22 11:03:37 +1000610 }
611
Rusty Russellb3f24692009-09-24 09:59:19 -0600612 hdr->mhdr.num_buffers = 0;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800613
614 /* Encode metadata header at front. */
615 if (vi->mergeable_rx_bufs)
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -0700616 sg_set_buf(vi->tx_sg, &hdr->mhdr, sizeof hdr->mhdr);
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800617 else
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -0700618 sg_set_buf(vi->tx_sg, &hdr->hdr, sizeof hdr->hdr);
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800619
Michael S. Tsirkin7bedc7d2012-10-16 23:56:14 +1030620 num_sg = skb_to_sgvec(skb, vi->tx_sg + 1, 0, skb->len) + 1;
621 return virtqueue_add_buf(vi->svq, vi->tx_sg, num_sg,
Rusty Russellf96fde42012-01-12 15:44:42 +1030622 0, skb, GFP_ATOMIC);
Rusty Russell11a3a152008-05-26 17:48:13 +1000623}
624
Stephen Hemminger424efe92009-08-31 19:50:51 +0000625static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
Rusty Russell99ffc692008-05-02 21:50:46 -0500626{
627 struct virtnet_info *vi = netdev_priv(dev);
Rusty Russell48925e32009-09-24 09:59:20 -0600628 int capacity;
Rusty Russell2cb9c6b2008-02-04 23:50:07 -0500629
Rusty Russell2cb9c6b2008-02-04 23:50:07 -0500630 /* Free up any pending old buffers before queueing new ones. */
631 free_old_xmit_skbs(vi);
Rusty Russell2cb9c6b2008-02-04 23:50:07 -0500632
Michael S. Tsirkin03f191b2009-10-28 04:03:38 -0700633 /* Try to transmit */
Rusty Russell48925e32009-09-24 09:59:20 -0600634 capacity = xmit_skb(vi, skb);
635
636 /* This can happen with OOM and indirect buffers. */
637 if (unlikely(capacity < 0)) {
Rick Jones2e57b792012-03-27 07:28:09 +0000638 if (likely(capacity == -ENOMEM)) {
Torsten Kaiser31304162012-04-09 05:14:15 +0000639 if (net_ratelimit())
Rusty Russell58eba97d2010-07-02 16:34:01 +0000640 dev_warn(&dev->dev,
641 "TX queue failure: out of memory\n");
Torsten Kaiser31304162012-04-09 05:14:15 +0000642 } else {
Rick Jones2e57b792012-03-27 07:28:09 +0000643 dev->stats.tx_fifo_errors++;
644 if (net_ratelimit())
Rusty Russell58eba97d2010-07-02 16:34:01 +0000645 dev_warn(&dev->dev,
646 "Unexpected TX queue failure: %d\n",
647 capacity);
Rusty Russell48925e32009-09-24 09:59:20 -0600648 }
Rusty Russell58eba97d2010-07-02 16:34:01 +0000649 dev->stats.tx_dropped++;
650 kfree_skb(skb);
651 return NETDEV_TX_OK;
Rusty Russell99ffc692008-05-02 21:50:46 -0500652 }
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300653 virtqueue_kick(vi->svq);
Michael S. Tsirkin03f191b2009-10-28 04:03:38 -0700654
Rusty Russell48925e32009-09-24 09:59:20 -0600655 /* Don't wait up for transmitted skbs to be freed. */
656 skb_orphan(skb);
657 nf_reset(skb);
Rusty Russell99ffc692008-05-02 21:50:46 -0500658
Rusty Russell48925e32009-09-24 09:59:20 -0600659 /* Apparently nice girls don't return TX_BUSY; stop the queue
660 * before it gets out of hand. Naturally, this wastes entries. */
661 if (capacity < 2+MAX_SKB_FRAGS) {
662 netif_stop_queue(dev);
Michael S. Tsirkin7a66f782011-05-20 02:11:23 +0300663 if (unlikely(!virtqueue_enable_cb_delayed(vi->svq))) {
Rusty Russell48925e32009-09-24 09:59:20 -0600664 /* More just got used, free them then recheck. */
Michael S. Tsirkin6ee57bc2012-10-16 23:56:14 +1030665 free_old_xmit_skbs(vi);
666 capacity = vi->svq->num_free;
Rusty Russell48925e32009-09-24 09:59:20 -0600667 if (capacity >= 2+MAX_SKB_FRAGS) {
668 netif_start_queue(dev);
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300669 virtqueue_disable_cb(vi->svq);
Rusty Russell48925e32009-09-24 09:59:20 -0600670 }
671 }
Rusty Russell99ffc692008-05-02 21:50:46 -0500672 }
Rusty Russell48925e32009-09-24 09:59:20 -0600673
674 return NETDEV_TX_OK;
Rusty Russell296f96f2007-10-22 11:03:37 +1000675}
676
Alex Williamson9c46f6d2009-02-04 16:36:34 -0800677static int virtnet_set_mac_address(struct net_device *dev, void *p)
678{
679 struct virtnet_info *vi = netdev_priv(dev);
680 struct virtio_device *vdev = vi->vdev;
Jiri Pirkof2f2c8b2012-06-29 05:10:06 +0000681 int ret;
Alex Williamson9c46f6d2009-02-04 16:36:34 -0800682
Jiri Pirkof2f2c8b2012-06-29 05:10:06 +0000683 ret = eth_mac_addr(dev, p);
684 if (ret)
685 return ret;
Alex Williamson9c46f6d2009-02-04 16:36:34 -0800686
Alex Williamson62994b22009-04-04 16:40:19 -0700687 if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
688 vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
689 dev->dev_addr, dev->addr_len);
Alex Williamson9c46f6d2009-02-04 16:36:34 -0800690
691 return 0;
692}
693
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000694static struct rtnl_link_stats64 *virtnet_stats(struct net_device *dev,
695 struct rtnl_link_stats64 *tot)
696{
697 struct virtnet_info *vi = netdev_priv(dev);
698 int cpu;
699 unsigned int start;
700
701 for_each_possible_cpu(cpu) {
Eric Dumazet58472a72012-02-13 06:53:41 +0000702 struct virtnet_stats *stats = per_cpu_ptr(vi->stats, cpu);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000703 u64 tpackets, tbytes, rpackets, rbytes;
704
705 do {
Kevin Groenevelde3906482012-07-21 06:30:50 +0000706 start = u64_stats_fetch_begin_bh(&stats->tx_syncp);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000707 tpackets = stats->tx_packets;
708 tbytes = stats->tx_bytes;
Kevin Groenevelde3906482012-07-21 06:30:50 +0000709 } while (u64_stats_fetch_retry_bh(&stats->tx_syncp, start));
Eric Dumazet83a27052012-06-05 22:35:24 +0000710
711 do {
Kevin Groenevelde3906482012-07-21 06:30:50 +0000712 start = u64_stats_fetch_begin_bh(&stats->rx_syncp);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000713 rpackets = stats->rx_packets;
714 rbytes = stats->rx_bytes;
Kevin Groenevelde3906482012-07-21 06:30:50 +0000715 } while (u64_stats_fetch_retry_bh(&stats->rx_syncp, start));
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000716
717 tot->rx_packets += rpackets;
718 tot->tx_packets += tpackets;
719 tot->rx_bytes += rbytes;
720 tot->tx_bytes += tbytes;
721 }
722
723 tot->tx_dropped = dev->stats.tx_dropped;
Rick Jones021ac8d2011-11-21 09:28:17 +0000724 tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000725 tot->rx_dropped = dev->stats.rx_dropped;
726 tot->rx_length_errors = dev->stats.rx_length_errors;
727 tot->rx_frame_errors = dev->stats.rx_frame_errors;
728
729 return tot;
730}
731
Amit Shahda74e892008-02-29 16:24:50 +0530732#ifdef CONFIG_NET_POLL_CONTROLLER
733static void virtnet_netpoll(struct net_device *dev)
734{
735 struct virtnet_info *vi = netdev_priv(dev);
736
737 napi_schedule(&vi->napi);
738}
739#endif
740
Rusty Russell296f96f2007-10-22 11:03:37 +1000741static int virtnet_open(struct net_device *dev)
742{
743 struct virtnet_info *vi = netdev_priv(dev);
744
Rusty Russellb2baed62011-12-29 00:42:38 +0000745 /* Make sure we have some buffers: if oom use wq. */
746 if (!try_fill_recv(vi, GFP_KERNEL))
Tejun Heo3b07e9c2012-08-20 14:51:24 -0700747 schedule_delayed_work(&vi->refill, 0);
Rusty Russellb2baed62011-12-29 00:42:38 +0000748
Bruce Rogers3e9d08e2011-02-10 11:03:31 -0800749 virtnet_napi_enable(vi);
Rusty Russell296f96f2007-10-22 11:03:37 +1000750 return 0;
751}
752
Alex Williamson2a41f712009-02-04 09:02:34 +0000753/*
754 * Send command via the control virtqueue and check status. Commands
755 * supported by the hypervisor, as indicated by feature bits, should
756 * never fail unless improperly formated.
757 */
758static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
759 struct scatterlist *data, int out, int in)
760{
Alex Williamson23e258e2009-05-01 17:27:56 +0000761 struct scatterlist *s, sg[VIRTNET_SEND_COMMAND_SG_MAX + 2];
Alex Williamson2a41f712009-02-04 09:02:34 +0000762 struct virtio_net_ctrl_hdr ctrl;
763 virtio_net_ctrl_ack status = ~0;
764 unsigned int tmp;
Alex Williamson23e258e2009-05-01 17:27:56 +0000765 int i;
Alex Williamson2a41f712009-02-04 09:02:34 +0000766
Alexander Beregalov0ee904c2009-04-11 14:50:23 +0000767 /* Caller should know better */
768 BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ||
769 (out + in > VIRTNET_SEND_COMMAND_SG_MAX));
Alex Williamson2a41f712009-02-04 09:02:34 +0000770
771 out++; /* Add header */
772 in++; /* Add return status */
773
774 ctrl.class = class;
775 ctrl.cmd = cmd;
776
777 sg_init_table(sg, out + in);
778
779 sg_set_buf(&sg[0], &ctrl, sizeof(ctrl));
Alex Williamson23e258e2009-05-01 17:27:56 +0000780 for_each_sg(data, s, out + in - 2, i)
781 sg_set_buf(&sg[i + 1], sg_virt(s), s->length);
Alex Williamson2a41f712009-02-04 09:02:34 +0000782 sg_set_buf(&sg[out + in - 1], &status, sizeof(status));
783
Rusty Russellf96fde42012-01-12 15:44:42 +1030784 BUG_ON(virtqueue_add_buf(vi->cvq, sg, out, in, vi, GFP_ATOMIC) < 0);
Alex Williamson2a41f712009-02-04 09:02:34 +0000785
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300786 virtqueue_kick(vi->cvq);
Alex Williamson2a41f712009-02-04 09:02:34 +0000787
788 /*
789 * Spin for a response, the kick causes an ioport write, trapping
790 * into the hypervisor, so the request should be handled immediately.
791 */
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +0300792 while (!virtqueue_get_buf(vi->cvq, &tmp))
Alex Williamson2a41f712009-02-04 09:02:34 +0000793 cpu_relax();
794
795 return status == VIRTIO_NET_OK;
796}
797
Jason Wang586d17c2012-04-11 20:43:52 +0000798static void virtnet_ack_link_announce(struct virtnet_info *vi)
799{
800 rtnl_lock();
801 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
802 VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL,
803 0, 0))
804 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
805 rtnl_unlock();
806}
807
Rusty Russell296f96f2007-10-22 11:03:37 +1000808static int virtnet_close(struct net_device *dev)
809{
810 struct virtnet_info *vi = netdev_priv(dev);
Rusty Russell296f96f2007-10-22 11:03:37 +1000811
Rusty Russellb2baed62011-12-29 00:42:38 +0000812 /* Make sure refill_work doesn't re-enable napi! */
813 cancel_delayed_work_sync(&vi->refill);
Rusty Russell296f96f2007-10-22 11:03:37 +1000814 napi_disable(&vi->napi);
815
Rusty Russell296f96f2007-10-22 11:03:37 +1000816 return 0;
817}
818
Alex Williamson2af76982009-02-04 09:02:40 +0000819static void virtnet_set_rx_mode(struct net_device *dev)
820{
821 struct virtnet_info *vi = netdev_priv(dev);
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000822 struct scatterlist sg[2];
Alex Williamson2af76982009-02-04 09:02:40 +0000823 u8 promisc, allmulti;
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000824 struct virtio_net_ctrl_mac *mac_data;
Jiri Pirkoccffad252009-05-22 23:22:17 +0000825 struct netdev_hw_addr *ha;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -0800826 int uc_count;
Jiri Pirko4cd24ea2010-02-08 04:30:35 +0000827 int mc_count;
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000828 void *buf;
829 int i;
Alex Williamson2af76982009-02-04 09:02:40 +0000830
831 /* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
832 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
833 return;
834
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000835 promisc = ((dev->flags & IFF_PROMISC) != 0);
836 allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
Alex Williamson2af76982009-02-04 09:02:40 +0000837
Alex Williamson23e258e2009-05-01 17:27:56 +0000838 sg_init_one(sg, &promisc, sizeof(promisc));
Alex Williamson2af76982009-02-04 09:02:40 +0000839
840 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
841 VIRTIO_NET_CTRL_RX_PROMISC,
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000842 sg, 1, 0))
Alex Williamson2af76982009-02-04 09:02:40 +0000843 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
844 promisc ? "en" : "dis");
845
Alex Williamson23e258e2009-05-01 17:27:56 +0000846 sg_init_one(sg, &allmulti, sizeof(allmulti));
Alex Williamson2af76982009-02-04 09:02:40 +0000847
848 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
849 VIRTIO_NET_CTRL_RX_ALLMULTI,
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000850 sg, 1, 0))
Alex Williamson2af76982009-02-04 09:02:40 +0000851 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
852 allmulti ? "en" : "dis");
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000853
Jiri Pirko32e7bfc2010-01-25 13:36:10 -0800854 uc_count = netdev_uc_count(dev);
Jiri Pirko4cd24ea2010-02-08 04:30:35 +0000855 mc_count = netdev_mc_count(dev);
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000856 /* MAC filter - use one buffer for both lists */
Jiri Pirko4cd24ea2010-02-08 04:30:35 +0000857 buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
858 (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
859 mac_data = buf;
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000860 if (!buf) {
861 dev_warn(&dev->dev, "No memory for MAC address buffer\n");
862 return;
863 }
864
Alex Williamson23e258e2009-05-01 17:27:56 +0000865 sg_init_table(sg, 2);
866
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000867 /* Store the unicast list and count in the front of the buffer */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -0800868 mac_data->entries = uc_count;
Jiri Pirkoccffad252009-05-22 23:22:17 +0000869 i = 0;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -0800870 netdev_for_each_uc_addr(ha, dev)
Jiri Pirkoccffad252009-05-22 23:22:17 +0000871 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000872
873 sg_set_buf(&sg[0], mac_data,
Jiri Pirko32e7bfc2010-01-25 13:36:10 -0800874 sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000875
876 /* multicast list and count fill the end */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -0800877 mac_data = (void *)&mac_data->macs[uc_count][0];
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000878
Jiri Pirko4cd24ea2010-02-08 04:30:35 +0000879 mac_data->entries = mc_count;
Jiri Pirko567ec872010-02-23 23:17:07 +0000880 i = 0;
Jiri Pirko22bedad32010-04-01 21:22:57 +0000881 netdev_for_each_mc_addr(ha, dev)
882 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000883
884 sg_set_buf(&sg[1], mac_data,
Jiri Pirko4cd24ea2010-02-08 04:30:35 +0000885 sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
Alex Williamsonf565a7c2009-02-04 09:02:45 +0000886
887 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
888 VIRTIO_NET_CTRL_MAC_TABLE_SET,
889 sg, 2, 0))
890 dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");
891
892 kfree(buf);
Alex Williamson2af76982009-02-04 09:02:40 +0000893}
894
Jiri Pirko8e586132011-12-08 19:52:37 -0500895static int virtnet_vlan_rx_add_vid(struct net_device *dev, u16 vid)
Alex Williamson0bde95692009-02-04 09:02:50 +0000896{
897 struct virtnet_info *vi = netdev_priv(dev);
898 struct scatterlist sg;
899
Alex Williamson23e258e2009-05-01 17:27:56 +0000900 sg_init_one(&sg, &vid, sizeof(vid));
Alex Williamson0bde95692009-02-04 09:02:50 +0000901
902 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
903 VIRTIO_NET_CTRL_VLAN_ADD, &sg, 1, 0))
904 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
Jiri Pirko8e586132011-12-08 19:52:37 -0500905 return 0;
Alex Williamson0bde95692009-02-04 09:02:50 +0000906}
907
Jiri Pirko8e586132011-12-08 19:52:37 -0500908static int virtnet_vlan_rx_kill_vid(struct net_device *dev, u16 vid)
Alex Williamson0bde95692009-02-04 09:02:50 +0000909{
910 struct virtnet_info *vi = netdev_priv(dev);
911 struct scatterlist sg;
912
Alex Williamson23e258e2009-05-01 17:27:56 +0000913 sg_init_one(&sg, &vid, sizeof(vid));
Alex Williamson0bde95692009-02-04 09:02:50 +0000914
915 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
916 VIRTIO_NET_CTRL_VLAN_DEL, &sg, 1, 0))
917 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
Jiri Pirko8e586132011-12-08 19:52:37 -0500918 return 0;
Alex Williamson0bde95692009-02-04 09:02:50 +0000919}
920
Rick Jones8f9f4662011-10-19 08:10:59 +0000921static void virtnet_get_ringparam(struct net_device *dev,
922 struct ethtool_ringparam *ring)
923{
924 struct virtnet_info *vi = netdev_priv(dev);
925
926 ring->rx_max_pending = virtqueue_get_vring_size(vi->rvq);
927 ring->tx_max_pending = virtqueue_get_vring_size(vi->svq);
928 ring->rx_pending = ring->rx_max_pending;
929 ring->tx_pending = ring->tx_max_pending;
930
931}
932
Rick Jones66846042011-11-14 14:17:08 +0000933
934static void virtnet_get_drvinfo(struct net_device *dev,
935 struct ethtool_drvinfo *info)
936{
937 struct virtnet_info *vi = netdev_priv(dev);
938 struct virtio_device *vdev = vi->vdev;
939
940 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
941 strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
942 strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
943
944}
945
Stephen Hemminger0fc0b732009-09-02 01:03:33 -0700946static const struct ethtool_ops virtnet_ethtool_ops = {
Rick Jones66846042011-11-14 14:17:08 +0000947 .get_drvinfo = virtnet_get_drvinfo,
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -0800948 .get_link = ethtool_op_get_link,
Rick Jones8f9f4662011-10-19 08:10:59 +0000949 .get_ringparam = virtnet_get_ringparam,
Herbert Xua9ea3fc2008-04-18 11:21:42 +0800950};
951
Mark McLoughlin39da5812008-11-26 13:58:11 +0000952#define MIN_MTU 68
953#define MAX_MTU 65535
954
955static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
956{
957 if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
958 return -EINVAL;
959 dev->mtu = new_mtu;
960 return 0;
961}
962
Stephen Hemminger76288b42009-01-06 10:44:22 -0800963static const struct net_device_ops virtnet_netdev = {
964 .ndo_open = virtnet_open,
965 .ndo_stop = virtnet_close,
966 .ndo_start_xmit = start_xmit,
967 .ndo_validate_addr = eth_validate_addr,
Alex Williamson9c46f6d2009-02-04 16:36:34 -0800968 .ndo_set_mac_address = virtnet_set_mac_address,
Alex Williamson2af76982009-02-04 09:02:40 +0000969 .ndo_set_rx_mode = virtnet_set_rx_mode,
Stephen Hemminger76288b42009-01-06 10:44:22 -0800970 .ndo_change_mtu = virtnet_change_mtu,
stephen hemminger3fa2a1d2011-06-15 06:36:29 +0000971 .ndo_get_stats64 = virtnet_stats,
Alex Williamson1824a982009-05-01 17:31:10 +0000972 .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
973 .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
Stephen Hemminger76288b42009-01-06 10:44:22 -0800974#ifdef CONFIG_NET_POLL_CONTROLLER
975 .ndo_poll_controller = virtnet_netpoll,
976#endif
977};
978
Jason Wang586d17c2012-04-11 20:43:52 +0000979static void virtnet_config_changed_work(struct work_struct *work)
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -0800980{
Jason Wang586d17c2012-04-11 20:43:52 +0000981 struct virtnet_info *vi =
982 container_of(work, struct virtnet_info, config_work);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -0800983 u16 v;
984
Jason Wang586d17c2012-04-11 20:43:52 +0000985 mutex_lock(&vi->config_lock);
986 if (!vi->config_enable)
987 goto done;
988
Sasha Levin77dd7692011-08-14 17:52:33 +0300989 if (virtio_config_val(vi->vdev, VIRTIO_NET_F_STATUS,
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -0800990 offsetof(struct virtio_net_config, status),
Sasha Levin77dd7692011-08-14 17:52:33 +0300991 &v) < 0)
Jason Wang586d17c2012-04-11 20:43:52 +0000992 goto done;
993
994 if (v & VIRTIO_NET_S_ANNOUNCE) {
Amerigo Wangee89bab2012-08-09 22:14:56 +0000995 netdev_notify_peers(vi->dev);
Jason Wang586d17c2012-04-11 20:43:52 +0000996 virtnet_ack_link_announce(vi);
997 }
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -0800998
999 /* Ignore unknown (future) status bits */
1000 v &= VIRTIO_NET_S_LINK_UP;
1001
1002 if (vi->status == v)
Jason Wang586d17c2012-04-11 20:43:52 +00001003 goto done;
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08001004
1005 vi->status = v;
1006
1007 if (vi->status & VIRTIO_NET_S_LINK_UP) {
1008 netif_carrier_on(vi->dev);
1009 netif_wake_queue(vi->dev);
1010 } else {
1011 netif_carrier_off(vi->dev);
1012 netif_stop_queue(vi->dev);
1013 }
Jason Wang586d17c2012-04-11 20:43:52 +00001014done:
1015 mutex_unlock(&vi->config_lock);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08001016}
1017
1018static void virtnet_config_changed(struct virtio_device *vdev)
1019{
1020 struct virtnet_info *vi = vdev->priv;
1021
Tejun Heo3b07e9c2012-08-20 14:51:24 -07001022 schedule_work(&vi->config_work);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08001023}
1024
Amit Shah3f9c10b2011-12-22 16:58:31 +05301025static int init_vqs(struct virtnet_info *vi)
1026{
1027 struct virtqueue *vqs[3];
1028 vq_callback_t *callbacks[] = { skb_recv_done, skb_xmit_done, NULL};
1029 const char *names[] = { "input", "output", "control" };
1030 int nvqs, err;
1031
1032 /* We expect two virtqueues, receive then send,
1033 * and optionally control. */
1034 nvqs = virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ? 3 : 2;
1035
1036 err = vi->vdev->config->find_vqs(vi->vdev, nvqs, vqs, callbacks, names);
1037 if (err)
1038 return err;
1039
1040 vi->rvq = vqs[0];
1041 vi->svq = vqs[1];
1042
1043 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)) {
1044 vi->cvq = vqs[2];
1045
1046 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
1047 vi->dev->features |= NETIF_F_HW_VLAN_FILTER;
1048 }
1049 return 0;
1050}
1051
Rusty Russell296f96f2007-10-22 11:03:37 +10001052static int virtnet_probe(struct virtio_device *vdev)
1053{
1054 int err;
Rusty Russell296f96f2007-10-22 11:03:37 +10001055 struct net_device *dev;
1056 struct virtnet_info *vi;
Rusty Russell296f96f2007-10-22 11:03:37 +10001057
1058 /* Allocate ourselves a network device with room for our info */
1059 dev = alloc_etherdev(sizeof(struct virtnet_info));
1060 if (!dev)
1061 return -ENOMEM;
1062
1063 /* Set up network device as normal. */
Jiri Pirkof2f2c8b2012-06-29 05:10:06 +00001064 dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
Stephen Hemminger76288b42009-01-06 10:44:22 -08001065 dev->netdev_ops = &virtnet_netdev;
Rusty Russell296f96f2007-10-22 11:03:37 +10001066 dev->features = NETIF_F_HIGHDMA;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001067
Herbert Xua9ea3fc2008-04-18 11:21:42 +08001068 SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
Rusty Russell296f96f2007-10-22 11:03:37 +10001069 SET_NETDEV_DEV(dev, &vdev->dev);
1070
1071 /* Do we support "hardware" checksums? */
Michał Mirosław98e778c2011-03-31 01:01:35 +00001072 if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
Rusty Russell296f96f2007-10-22 11:03:37 +10001073 /* This opens up the world of extra features. */
Michał Mirosław98e778c2011-03-31 01:01:35 +00001074 dev->hw_features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
1075 if (csum)
1076 dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
1077
1078 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
1079 dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
Rusty Russell34a48572008-02-04 23:50:02 -05001080 | NETIF_F_TSO_ECN | NETIF_F_TSO6;
1081 }
Rusty Russell5539ae962008-05-02 21:50:46 -05001082 /* Individual feature bits: what can host handle? */
Michał Mirosław98e778c2011-03-31 01:01:35 +00001083 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
1084 dev->hw_features |= NETIF_F_TSO;
1085 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
1086 dev->hw_features |= NETIF_F_TSO6;
1087 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
1088 dev->hw_features |= NETIF_F_TSO_ECN;
1089 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
1090 dev->hw_features |= NETIF_F_UFO;
1091
1092 if (gso)
1093 dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
1094 /* (!csum && gso) case will be fixed by register_netdev() */
Rusty Russell296f96f2007-10-22 11:03:37 +10001095 }
1096
1097 /* Configuration may specify what MAC to use. Otherwise random. */
Sasha Levin77dd7692011-08-14 17:52:33 +03001098 if (virtio_config_val_len(vdev, VIRTIO_NET_F_MAC,
Rusty Russella586d4f2008-02-04 23:49:56 -05001099 offsetof(struct virtio_net_config, mac),
Sasha Levin77dd7692011-08-14 17:52:33 +03001100 dev->dev_addr, dev->addr_len) < 0)
Danny Kukawkaf2cedb62012-02-15 06:45:39 +00001101 eth_hw_addr_random(dev);
Rusty Russell296f96f2007-10-22 11:03:37 +10001102
1103 /* Set up our device-specific information */
1104 vi = netdev_priv(dev);
Dor Laor6c0cd7c2007-12-16 15:19:43 +02001105 netif_napi_add(dev, &vi->napi, virtnet_poll, napi_weight);
Rusty Russell296f96f2007-10-22 11:03:37 +10001106 vi->dev = dev;
1107 vi->vdev = vdev;
Christian Borntraegerd9d5dcc2008-02-18 10:02:51 +01001108 vdev->priv = vi;
Rusty Russellfb6813f2008-07-25 12:06:01 -05001109 vi->pages = NULL;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001110 vi->stats = alloc_percpu(struct virtnet_stats);
1111 err = -ENOMEM;
1112 if (vi->stats == NULL)
1113 goto free;
1114
Rusty Russell3161e452009-08-26 12:22:32 -07001115 INIT_DELAYED_WORK(&vi->refill, refill_work);
Jason Wang586d17c2012-04-11 20:43:52 +00001116 mutex_init(&vi->config_lock);
1117 vi->config_enable = true;
1118 INIT_WORK(&vi->config_work, virtnet_config_changed_work);
Michael S. Tsirkin5e01d2f2010-04-07 21:01:41 -07001119 sg_init_table(vi->rx_sg, ARRAY_SIZE(vi->rx_sg));
1120 sg_init_table(vi->tx_sg, ARRAY_SIZE(vi->tx_sg));
Rusty Russell296f96f2007-10-22 11:03:37 +10001121
Herbert Xu97402b92008-04-18 11:24:27 +08001122 /* If we can receive ANY GSO packets, we must allocate large ones. */
Joe Perches8e95a202009-12-03 07:58:21 +00001123 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
1124 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
1125 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
Herbert Xu97402b92008-04-18 11:24:27 +08001126 vi->big_packets = true;
1127
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001128 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
1129 vi->mergeable_rx_bufs = true;
1130
Amit Shah3f9c10b2011-12-22 16:58:31 +05301131 err = init_vqs(vi);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06001132 if (err)
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001133 goto free_stats;
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06001134
Rusty Russell296f96f2007-10-22 11:03:37 +10001135 err = register_netdev(dev);
1136 if (err) {
1137 pr_debug("virtio_net: registering device failed\n");
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06001138 goto free_vqs;
Rusty Russell296f96f2007-10-22 11:03:37 +10001139 }
Rusty Russellb3369c12008-02-04 23:50:02 -05001140
1141 /* Last of all, set up some receive buffers. */
Rusty Russell3161e452009-08-26 12:22:32 -07001142 try_fill_recv(vi, GFP_KERNEL);
Rusty Russellb3369c12008-02-04 23:50:02 -05001143
1144 /* If we didn't even get one input buffer, we're useless. */
1145 if (vi->num == 0) {
1146 err = -ENOMEM;
1147 goto unregister;
1148 }
1149
Jason Wang167c25e2010-11-10 14:45:41 +00001150 /* Assume link up if device can't report link status,
1151 otherwise get link status from config. */
1152 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
1153 netif_carrier_off(dev);
Tejun Heo3b07e9c2012-08-20 14:51:24 -07001154 schedule_work(&vi->config_work);
Jason Wang167c25e2010-11-10 14:45:41 +00001155 } else {
1156 vi->status = VIRTIO_NET_S_LINK_UP;
1157 netif_carrier_on(dev);
1158 }
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08001159
Rusty Russell296f96f2007-10-22 11:03:37 +10001160 pr_debug("virtnet: registered device %s\n", dev->name);
Rusty Russell296f96f2007-10-22 11:03:37 +10001161 return 0;
1162
Rusty Russellb3369c12008-02-04 23:50:02 -05001163unregister:
1164 unregister_netdev(dev);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06001165free_vqs:
1166 vdev->config->del_vqs(vdev);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001167free_stats:
1168 free_percpu(vi->stats);
Rusty Russell296f96f2007-10-22 11:03:37 +10001169free:
1170 free_netdev(dev);
1171 return err;
1172}
1173
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001174static void free_unused_bufs(struct virtnet_info *vi)
1175{
1176 void *buf;
1177 while (1) {
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +03001178 buf = virtqueue_detach_unused_buf(vi->svq);
Shirley Ma830a8a92010-02-08 14:14:42 +00001179 if (!buf)
1180 break;
1181 dev_kfree_skb(buf);
1182 }
1183 while (1) {
Michael S. Tsirkin1915a7122010-04-12 16:19:04 +03001184 buf = virtqueue_detach_unused_buf(vi->rvq);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001185 if (!buf)
1186 break;
1187 if (vi->mergeable_rx_bufs || vi->big_packets)
1188 give_pages(vi, buf);
1189 else
1190 dev_kfree_skb(buf);
1191 --vi->num;
1192 }
1193 BUG_ON(vi->num != 0);
1194}
1195
Amit Shah04486ed2011-12-22 16:58:32 +05301196static void remove_vq_common(struct virtnet_info *vi)
Rusty Russell296f96f2007-10-22 11:03:37 +10001197{
Amit Shah04486ed2011-12-22 16:58:32 +05301198 vi->vdev->config->reset(vi->vdev);
Shirley Ma830a8a92010-02-08 14:14:42 +00001199
1200 /* Free unused buffers in both send and recv, if any. */
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001201 free_unused_bufs(vi);
Rusty Russellfb6813f2008-07-25 12:06:01 -05001202
Amit Shah04486ed2011-12-22 16:58:32 +05301203 vi->vdev->config->del_vqs(vi->vdev);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06001204
Rusty Russellfb6813f2008-07-25 12:06:01 -05001205 while (vi->pages)
1206 __free_pages(get_a_page(vi, GFP_KERNEL), 0);
Amit Shah04486ed2011-12-22 16:58:32 +05301207}
1208
1209static void __devexit virtnet_remove(struct virtio_device *vdev)
1210{
1211 struct virtnet_info *vi = vdev->priv;
1212
Jason Wang586d17c2012-04-11 20:43:52 +00001213 /* Prevent config work handler from accessing the device. */
1214 mutex_lock(&vi->config_lock);
1215 vi->config_enable = false;
1216 mutex_unlock(&vi->config_lock);
1217
Amit Shah04486ed2011-12-22 16:58:32 +05301218 unregister_netdev(vi->dev);
1219
1220 remove_vq_common(vi);
Rusty Russellfb6813f2008-07-25 12:06:01 -05001221
Jason Wang586d17c2012-04-11 20:43:52 +00001222 flush_work(&vi->config_work);
1223
Krishna Kumar2e66f552011-07-20 03:56:02 +00001224 free_percpu(vi->stats);
Rusty Russell74b25532007-11-19 11:20:42 -05001225 free_netdev(vi->dev);
Rusty Russell296f96f2007-10-22 11:03:37 +10001226}
1227
Amit Shah0741bcb2011-12-22 16:58:33 +05301228#ifdef CONFIG_PM
1229static int virtnet_freeze(struct virtio_device *vdev)
1230{
1231 struct virtnet_info *vi = vdev->priv;
1232
Jason Wang586d17c2012-04-11 20:43:52 +00001233 /* Prevent config work handler from accessing the device */
1234 mutex_lock(&vi->config_lock);
1235 vi->config_enable = false;
1236 mutex_unlock(&vi->config_lock);
1237
Amit Shah0741bcb2011-12-22 16:58:33 +05301238 netif_device_detach(vi->dev);
1239 cancel_delayed_work_sync(&vi->refill);
1240
1241 if (netif_running(vi->dev))
1242 napi_disable(&vi->napi);
1243
1244 remove_vq_common(vi);
1245
Jason Wang586d17c2012-04-11 20:43:52 +00001246 flush_work(&vi->config_work);
1247
Amit Shah0741bcb2011-12-22 16:58:33 +05301248 return 0;
1249}
1250
1251static int virtnet_restore(struct virtio_device *vdev)
1252{
1253 struct virtnet_info *vi = vdev->priv;
1254 int err;
1255
1256 err = init_vqs(vi);
1257 if (err)
1258 return err;
1259
1260 if (netif_running(vi->dev))
1261 virtnet_napi_enable(vi);
1262
1263 netif_device_attach(vi->dev);
1264
1265 if (!try_fill_recv(vi, GFP_KERNEL))
Tejun Heo3b07e9c2012-08-20 14:51:24 -07001266 schedule_delayed_work(&vi->refill, 0);
Amit Shah0741bcb2011-12-22 16:58:33 +05301267
Jason Wang586d17c2012-04-11 20:43:52 +00001268 mutex_lock(&vi->config_lock);
1269 vi->config_enable = true;
1270 mutex_unlock(&vi->config_lock);
1271
Amit Shah0741bcb2011-12-22 16:58:33 +05301272 return 0;
1273}
1274#endif
1275
Rusty Russell296f96f2007-10-22 11:03:37 +10001276static struct virtio_device_id id_table[] = {
1277 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
1278 { 0 },
1279};
1280
Rusty Russellc45a6812008-05-02 21:50:50 -05001281static unsigned int features[] = {
Mark McLoughlin5e4fe5c2008-07-08 17:10:42 +10001282 VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
1283 VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
Rusty Russellc45a6812008-05-02 21:50:50 -05001284 VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
Herbert Xu97402b92008-04-18 11:24:27 +08001285 VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
Sridhar Samudrala5c516752009-07-14 14:21:02 +00001286 VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
Alex Williamson2a41f712009-02-04 09:02:34 +00001287 VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
Alex Williamson0bde95692009-02-04 09:02:50 +00001288 VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
Jason Wang586d17c2012-04-11 20:43:52 +00001289 VIRTIO_NET_F_GUEST_ANNOUNCE,
Rusty Russellc45a6812008-05-02 21:50:50 -05001290};
1291
Uwe Kleine-König22402522009-11-05 01:32:44 -08001292static struct virtio_driver virtio_net_driver = {
Rusty Russellc45a6812008-05-02 21:50:50 -05001293 .feature_table = features,
1294 .feature_table_size = ARRAY_SIZE(features),
Rusty Russell296f96f2007-10-22 11:03:37 +10001295 .driver.name = KBUILD_MODNAME,
1296 .driver.owner = THIS_MODULE,
1297 .id_table = id_table,
1298 .probe = virtnet_probe,
1299 .remove = __devexit_p(virtnet_remove),
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08001300 .config_changed = virtnet_config_changed,
Amit Shah0741bcb2011-12-22 16:58:33 +05301301#ifdef CONFIG_PM
1302 .freeze = virtnet_freeze,
1303 .restore = virtnet_restore,
1304#endif
Rusty Russell296f96f2007-10-22 11:03:37 +10001305};
1306
1307static int __init init(void)
1308{
Uwe Kleine-König22402522009-11-05 01:32:44 -08001309 return register_virtio_driver(&virtio_net_driver);
Rusty Russell296f96f2007-10-22 11:03:37 +10001310}
1311
1312static void __exit fini(void)
1313{
Uwe Kleine-König22402522009-11-05 01:32:44 -08001314 unregister_virtio_driver(&virtio_net_driver);
Rusty Russell296f96f2007-10-22 11:03:37 +10001315}
1316module_init(init);
1317module_exit(fini);
1318
1319MODULE_DEVICE_TABLE(virtio, id_table);
1320MODULE_DESCRIPTION("Virtio network driver");
1321MODULE_LICENSE("GPL");