blob: 9f66de9c057299abb9859ee293aad9f6056d478e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
76#include <asm/system.h>
77#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080078#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#include <linux/cpu.h>
80#include <linux/types.h>
81#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000082#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090083#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070084#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080085#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070086#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070095#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070096#include <linux/notifier.h>
97#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020098#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070099#include <net/sock.h>
100#include <linux/rtnetlink.h>
101#include <linux/proc_fs.h>
102#include <linux/seq_file.h>
103#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104#include <net/dst.h>
105#include <net/pkt_sched.h>
106#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000107#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <linux/highmem.h>
109#include <linux/init.h>
110#include <linux/kmod.h>
111#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <linux/netpoll.h>
113#include <linux/rcupdate.h>
114#include <linux/delay.h>
Johannes Berg295f4a12007-04-26 20:43:56 -0700115#include <net/wext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500118#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700119#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700120#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700121#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700122#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700123#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700124#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700125#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700126#include <linux/ipv6.h>
127#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700128#include <linux/jhash.h>
129#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700130#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900131#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900132#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000133#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700134#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000135#include <linux/cpu_rmap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700137#include "net-sysfs.h"
138
Herbert Xud565b0a2008-12-15 23:38:52 -0800139/* Instead of increasing this, you should create a hash table. */
140#define MAX_GRO_SKBS 8
141
Herbert Xu5d38a072009-01-04 16:13:40 -0800142/* This should be increased if a protocol with a bigger head is added. */
143#define GRO_MAX_HEAD (MAX_HEADER + 128)
144
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145/*
146 * The list of packet types we will receive (as opposed to discard)
147 * and the routines to invoke.
148 *
149 * Why 16. Because with 16 the only overlap we get on a hash of the
150 * low nibble of the protocol value is RARP/SNAP/X.25.
151 *
152 * NOTE: That is no longer true with the addition of VLAN tags. Not
153 * sure which should go first, but I bet it won't make much
154 * difference if we are running VLANs. The good news is that
155 * this protocol won't be in the list unless compiled in, so
Stephen Hemminger3041a062006-05-26 13:25:24 -0700156 * the average user (w/out VLANs) will not be adversely affected.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 * --BLG
158 *
159 * 0800 IP
160 * 8100 802.1Q VLAN
161 * 0001 802.3
162 * 0002 AX.25
163 * 0004 802.2
164 * 8035 RARP
165 * 0005 SNAP
166 * 0805 X.25
167 * 0806 ARP
168 * 8137 IPX
169 * 0009 Localtalk
170 * 86DD IPv6
171 */
172
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800173#define PTYPE_HASH_SIZE (16)
174#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
175
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176static DEFINE_SPINLOCK(ptype_lock);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800177static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -0700178static struct list_head ptype_all __read_mostly; /* Taps */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700181 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 * semaphore.
183 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800184 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 *
186 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700187 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188 * actual updates. This allows pure readers to access the list even
189 * while a writer is preparing to update it.
190 *
191 * To put it another way, dev_base_lock is held for writing only to
192 * protect against pure readers; the rtnl semaphore provides the
193 * protection against other writers.
194 *
195 * See, for example usages, register_netdevice() and
196 * unregister_netdevice(), which must be called with the rtnl
197 * semaphore held.
198 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200EXPORT_SYMBOL(dev_base_lock);
201
Eric W. Biederman881d9662007-09-17 11:56:21 -0700202static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203{
204 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
stephen hemminger08e98972009-11-10 07:20:34 +0000205 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206}
207
Eric W. Biederman881d9662007-09-17 11:56:21 -0700208static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700210 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211}
212
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000213static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000214{
215#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000216 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000217#endif
218}
219
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000220static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000221{
222#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000223 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000224#endif
225}
226
Eric W. Biedermance286d32007-09-12 13:53:49 +0200227/* Device list insertion */
228static int list_netdevice(struct net_device *dev)
229{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900230 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200231
232 ASSERT_RTNL();
233
234 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800235 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000236 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000237 hlist_add_head_rcu(&dev->index_hlist,
238 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200239 write_unlock_bh(&dev_base_lock);
240 return 0;
241}
242
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000243/* Device list removal
244 * caller must respect a RCU grace period before freeing/reusing dev
245 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200246static void unlist_netdevice(struct net_device *dev)
247{
248 ASSERT_RTNL();
249
250 /* Unlink dev from the device chain */
251 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800252 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000253 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000254 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200255 write_unlock_bh(&dev_base_lock);
256}
257
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258/*
259 * Our notifier list
260 */
261
Alan Sternf07d5b92006-05-09 15:23:03 -0700262static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263
264/*
265 * Device drivers call our routines to queue packets here. We empty the
266 * queue in the local softnet handler.
267 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700268
Eric Dumazet9958da02010-04-17 04:17:02 +0000269DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700270EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271
David S. Millercf508b12008-07-22 14:16:42 -0700272#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700273/*
David S. Millerc773e842008-07-08 23:13:53 -0700274 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700275 * according to dev->type
276 */
277static const unsigned short netdev_lock_type[] =
278 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
279 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
280 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
281 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
282 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
283 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
284 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
285 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
286 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
287 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
288 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
289 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
290 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800291 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
Dmitry Eremin-Solenikov929122c2009-08-14 20:00:20 +0400292 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
Sergey Lapinfcb94e42009-06-08 12:18:47 +0000293 ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700294
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700295static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700296 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
297 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
298 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
299 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
300 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
301 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
302 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
303 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
304 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
305 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
306 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
307 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
308 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800309 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
Dmitry Eremin-Solenikov929122c2009-08-14 20:00:20 +0400310 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
Sergey Lapinfcb94e42009-06-08 12:18:47 +0000311 "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700312
313static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700314static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700315
316static inline unsigned short netdev_lock_pos(unsigned short dev_type)
317{
318 int i;
319
320 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
321 if (netdev_lock_type[i] == dev_type)
322 return i;
323 /* the last key is used by default */
324 return ARRAY_SIZE(netdev_lock_type) - 1;
325}
326
David S. Millercf508b12008-07-22 14:16:42 -0700327static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
328 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700329{
330 int i;
331
332 i = netdev_lock_pos(dev_type);
333 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
334 netdev_lock_name[i]);
335}
David S. Millercf508b12008-07-22 14:16:42 -0700336
337static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
338{
339 int i;
340
341 i = netdev_lock_pos(dev->type);
342 lockdep_set_class_and_name(&dev->addr_list_lock,
343 &netdev_addr_lock_key[i],
344 netdev_lock_name[i]);
345}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700346#else
David S. Millercf508b12008-07-22 14:16:42 -0700347static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
348 unsigned short dev_type)
349{
350}
351static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700352{
353}
354#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355
356/*******************************************************************************
357
358 Protocol management and registration routines
359
360*******************************************************************************/
361
362/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 * Add a protocol ID to the list. Now that the input handler is
364 * smarter we can dispense with all the messy stuff that used to be
365 * here.
366 *
367 * BEWARE!!! Protocol handlers, mangling input packets,
368 * MUST BE last in hash buckets and checking protocol handlers
369 * MUST start from promiscuous ptype_all chain in net_bh.
370 * It is true now, do not change it.
371 * Explanation follows: if protocol handler, mangling packet, will
372 * be the first on list, it is not able to sense, that packet
373 * is cloned and should be copied-on-write, so that it will
374 * change it and subsequent readers will get broken packet.
375 * --ANK (980803)
376 */
377
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000378static inline struct list_head *ptype_head(const struct packet_type *pt)
379{
380 if (pt->type == htons(ETH_P_ALL))
381 return &ptype_all;
382 else
383 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
384}
385
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386/**
387 * dev_add_pack - add packet handler
388 * @pt: packet type declaration
389 *
390 * Add a protocol handler to the networking stack. The passed &packet_type
391 * is linked into kernel lists and may not be freed until it has been
392 * removed from the kernel lists.
393 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900394 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 * guarantee all CPU's that are in middle of receiving packets
396 * will see the new packet type (until the next received packet).
397 */
398
399void dev_add_pack(struct packet_type *pt)
400{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000401 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000403 spin_lock(&ptype_lock);
404 list_add_rcu(&pt->list, head);
405 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700407EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409/**
410 * __dev_remove_pack - remove packet handler
411 * @pt: packet type declaration
412 *
413 * Remove a protocol handler that was previously added to the kernel
414 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
415 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900416 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 *
418 * The packet type might still be in use by receivers
419 * and must not be freed until after all the CPU's have gone
420 * through a quiescent state.
421 */
422void __dev_remove_pack(struct packet_type *pt)
423{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000424 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 struct packet_type *pt1;
426
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000427 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428
429 list_for_each_entry(pt1, head, list) {
430 if (pt == pt1) {
431 list_del_rcu(&pt->list);
432 goto out;
433 }
434 }
435
436 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
437out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000438 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700440EXPORT_SYMBOL(__dev_remove_pack);
441
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442/**
443 * dev_remove_pack - remove packet handler
444 * @pt: packet type declaration
445 *
446 * Remove a protocol handler that was previously added to the kernel
447 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
448 * from the kernel lists and can be freed or reused once this function
449 * returns.
450 *
451 * This call sleeps to guarantee that no CPU is looking at the packet
452 * type after return.
453 */
454void dev_remove_pack(struct packet_type *pt)
455{
456 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900457
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458 synchronize_net();
459}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700460EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461
462/******************************************************************************
463
464 Device Boot-time Settings Routines
465
466*******************************************************************************/
467
468/* Boot time configuration table */
469static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
470
471/**
472 * netdev_boot_setup_add - add new setup entry
473 * @name: name of the device
474 * @map: configured settings for the device
475 *
476 * Adds new setup entry to the dev_boot_setup list. The function
477 * returns 0 on error and 1 on success. This is a generic routine to
478 * all netdevices.
479 */
480static int netdev_boot_setup_add(char *name, struct ifmap *map)
481{
482 struct netdev_boot_setup *s;
483 int i;
484
485 s = dev_boot_setup;
486 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
487 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
488 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700489 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 memcpy(&s[i].map, map, sizeof(s[i].map));
491 break;
492 }
493 }
494
495 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
496}
497
498/**
499 * netdev_boot_setup_check - check boot time settings
500 * @dev: the netdevice
501 *
502 * Check boot time settings for the device.
503 * The found settings are set for the device to be used
504 * later in the device probing.
505 * Returns 0 if no settings found, 1 if they are.
506 */
507int netdev_boot_setup_check(struct net_device *dev)
508{
509 struct netdev_boot_setup *s = dev_boot_setup;
510 int i;
511
512 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
513 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700514 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 dev->irq = s[i].map.irq;
516 dev->base_addr = s[i].map.base_addr;
517 dev->mem_start = s[i].map.mem_start;
518 dev->mem_end = s[i].map.mem_end;
519 return 1;
520 }
521 }
522 return 0;
523}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700524EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525
526
527/**
528 * netdev_boot_base - get address from boot time settings
529 * @prefix: prefix for network device
530 * @unit: id for network device
531 *
532 * Check boot time settings for the base address of device.
533 * The found settings are set for the device to be used
534 * later in the device probing.
535 * Returns 0 if no settings found.
536 */
537unsigned long netdev_boot_base(const char *prefix, int unit)
538{
539 const struct netdev_boot_setup *s = dev_boot_setup;
540 char name[IFNAMSIZ];
541 int i;
542
543 sprintf(name, "%s%d", prefix, unit);
544
545 /*
546 * If device already registered then return base of 1
547 * to indicate not to probe for this interface
548 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700549 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 return 1;
551
552 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
553 if (!strcmp(name, s[i].name))
554 return s[i].map.base_addr;
555 return 0;
556}
557
558/*
559 * Saves at boot time configured settings for any netdevice.
560 */
561int __init netdev_boot_setup(char *str)
562{
563 int ints[5];
564 struct ifmap map;
565
566 str = get_options(str, ARRAY_SIZE(ints), ints);
567 if (!str || !*str)
568 return 0;
569
570 /* Save settings */
571 memset(&map, 0, sizeof(map));
572 if (ints[0] > 0)
573 map.irq = ints[1];
574 if (ints[0] > 1)
575 map.base_addr = ints[2];
576 if (ints[0] > 2)
577 map.mem_start = ints[3];
578 if (ints[0] > 3)
579 map.mem_end = ints[4];
580
581 /* Add new entry to the list */
582 return netdev_boot_setup_add(str, &map);
583}
584
585__setup("netdev=", netdev_boot_setup);
586
587/*******************************************************************************
588
589 Device Interface Subroutines
590
591*******************************************************************************/
592
593/**
594 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700595 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 * @name: name to find
597 *
598 * Find an interface by name. Must be called under RTNL semaphore
599 * or @dev_base_lock. If the name is found a pointer to the device
600 * is returned. If the name is not found then %NULL is returned. The
601 * reference counters are not incremented so the caller must be
602 * careful with locks.
603 */
604
Eric W. Biederman881d9662007-09-17 11:56:21 -0700605struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606{
607 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700608 struct net_device *dev;
609 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700611 hlist_for_each_entry(dev, p, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612 if (!strncmp(dev->name, name, IFNAMSIZ))
613 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700614
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 return NULL;
616}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700617EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618
619/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000620 * dev_get_by_name_rcu - find a device by its name
621 * @net: the applicable net namespace
622 * @name: name to find
623 *
624 * Find an interface by name.
625 * If the name is found a pointer to the device is returned.
626 * If the name is not found then %NULL is returned.
627 * The reference counters are not incremented so the caller must be
628 * careful with locks. The caller must hold RCU lock.
629 */
630
631struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
632{
633 struct hlist_node *p;
634 struct net_device *dev;
635 struct hlist_head *head = dev_name_hash(net, name);
636
637 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
638 if (!strncmp(dev->name, name, IFNAMSIZ))
639 return dev;
640
641 return NULL;
642}
643EXPORT_SYMBOL(dev_get_by_name_rcu);
644
645/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700647 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 * @name: name to find
649 *
650 * Find an interface by name. This can be called from any
651 * context and does its own locking. The returned handle has
652 * the usage count incremented and the caller must use dev_put() to
653 * release it when it is no longer needed. %NULL is returned if no
654 * matching device is found.
655 */
656
Eric W. Biederman881d9662007-09-17 11:56:21 -0700657struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658{
659 struct net_device *dev;
660
Eric Dumazet72c95282009-10-30 07:11:27 +0000661 rcu_read_lock();
662 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 if (dev)
664 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000665 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 return dev;
667}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700668EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669
670/**
671 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700672 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 * @ifindex: index of device
674 *
675 * Search for an interface by index. Returns %NULL if the device
676 * is not found or a pointer to the device. The device has not
677 * had its reference counter increased so the caller must be careful
678 * about locking. The caller must hold either the RTNL semaphore
679 * or @dev_base_lock.
680 */
681
Eric W. Biederman881d9662007-09-17 11:56:21 -0700682struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683{
684 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700685 struct net_device *dev;
686 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700688 hlist_for_each_entry(dev, p, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 if (dev->ifindex == ifindex)
690 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700691
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692 return NULL;
693}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700694EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000696/**
697 * dev_get_by_index_rcu - find a device by its ifindex
698 * @net: the applicable net namespace
699 * @ifindex: index of device
700 *
701 * Search for an interface by index. Returns %NULL if the device
702 * is not found or a pointer to the device. The device has not
703 * had its reference counter increased so the caller must be careful
704 * about locking. The caller must hold RCU lock.
705 */
706
707struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
708{
709 struct hlist_node *p;
710 struct net_device *dev;
711 struct hlist_head *head = dev_index_hash(net, ifindex);
712
713 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
714 if (dev->ifindex == ifindex)
715 return dev;
716
717 return NULL;
718}
719EXPORT_SYMBOL(dev_get_by_index_rcu);
720
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721
722/**
723 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700724 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 * @ifindex: index of device
726 *
727 * Search for an interface by index. Returns NULL if the device
728 * is not found or a pointer to the device. The device returned has
729 * had a reference added and the pointer is safe until the user calls
730 * dev_put to indicate they have finished with it.
731 */
732
Eric W. Biederman881d9662007-09-17 11:56:21 -0700733struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734{
735 struct net_device *dev;
736
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000737 rcu_read_lock();
738 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 if (dev)
740 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000741 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742 return dev;
743}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700744EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745
746/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000747 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700748 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 * @type: media type of device
750 * @ha: hardware address
751 *
752 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800753 * is not found or a pointer to the device.
754 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000755 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 * and the caller must therefore be careful about locking
757 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758 */
759
Eric Dumazet941666c2010-12-05 01:23:53 +0000760struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
761 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762{
763 struct net_device *dev;
764
Eric Dumazet941666c2010-12-05 01:23:53 +0000765 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 if (dev->type == type &&
767 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700768 return dev;
769
770 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771}
Eric Dumazet941666c2010-12-05 01:23:53 +0000772EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300773
Eric W. Biederman881d9662007-09-17 11:56:21 -0700774struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700775{
776 struct net_device *dev;
777
778 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700779 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700780 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700781 return dev;
782
783 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700784}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700785EXPORT_SYMBOL(__dev_getfirstbyhwtype);
786
Eric W. Biederman881d9662007-09-17 11:56:21 -0700787struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000789 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000791 rcu_read_lock();
792 for_each_netdev_rcu(net, dev)
793 if (dev->type == type) {
794 dev_hold(dev);
795 ret = dev;
796 break;
797 }
798 rcu_read_unlock();
799 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801EXPORT_SYMBOL(dev_getfirstbyhwtype);
802
803/**
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000804 * dev_get_by_flags_rcu - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700805 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806 * @if_flags: IFF_* values
807 * @mask: bitmask of bits in if_flags to check
808 *
809 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000810 * is not found or a pointer to the device. Must be called inside
811 * rcu_read_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812 */
813
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000814struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700815 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700817 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818
Pavel Emelianov7562f872007-05-03 15:13:45 -0700819 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800820 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700822 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 break;
824 }
825 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700826 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827}
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000828EXPORT_SYMBOL(dev_get_by_flags_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829
830/**
831 * dev_valid_name - check if name is okay for network device
832 * @name: name string
833 *
834 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700835 * to allow sysfs to work. We also disallow any kind of
836 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837 */
Mitch Williamsc2373ee2005-11-09 10:34:45 -0800838int dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700840 if (*name == '\0')
841 return 0;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700842 if (strlen(name) >= IFNAMSIZ)
843 return 0;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700844 if (!strcmp(name, ".") || !strcmp(name, ".."))
845 return 0;
846
847 while (*name) {
848 if (*name == '/' || isspace(*name))
849 return 0;
850 name++;
851 }
852 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700854EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855
856/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200857 * __dev_alloc_name - allocate a name for a device
858 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200860 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861 *
862 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700863 * id. It scans list of devices to build up a free map, then chooses
864 * the first empty slot. The caller must hold the dev_base or rtnl lock
865 * while allocating the name and adding the device in order to avoid
866 * duplicates.
867 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
868 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869 */
870
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200871static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872{
873 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 const char *p;
875 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700876 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877 struct net_device *d;
878
879 p = strnchr(name, IFNAMSIZ-1, '%');
880 if (p) {
881 /*
882 * Verify the string as this thing may have come from
883 * the user. There must be either one "%d" and no other "%"
884 * characters.
885 */
886 if (p[1] != 'd' || strchr(p + 2, '%'))
887 return -EINVAL;
888
889 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700890 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891 if (!inuse)
892 return -ENOMEM;
893
Eric W. Biederman881d9662007-09-17 11:56:21 -0700894 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895 if (!sscanf(d->name, name, &i))
896 continue;
897 if (i < 0 || i >= max_netdevices)
898 continue;
899
900 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200901 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 if (!strncmp(buf, d->name, IFNAMSIZ))
903 set_bit(i, inuse);
904 }
905
906 i = find_first_zero_bit(inuse, max_netdevices);
907 free_page((unsigned long) inuse);
908 }
909
Octavian Purdilad9031022009-11-18 02:36:59 +0000910 if (buf != name)
911 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200912 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914
915 /* It is possible to run out of possible slots
916 * when the name is long and there isn't enough space left
917 * for the digits, or if all bits are used.
918 */
919 return -ENFILE;
920}
921
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200922/**
923 * dev_alloc_name - allocate a name for a device
924 * @dev: device
925 * @name: name format string
926 *
927 * Passed a format string - eg "lt%d" it will try and find a suitable
928 * id. It scans list of devices to build up a free map, then chooses
929 * the first empty slot. The caller must hold the dev_base or rtnl lock
930 * while allocating the name and adding the device in order to avoid
931 * duplicates.
932 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
933 * Returns the number of the unit assigned or a negative errno code.
934 */
935
936int dev_alloc_name(struct net_device *dev, const char *name)
937{
938 char buf[IFNAMSIZ];
939 struct net *net;
940 int ret;
941
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900942 BUG_ON(!dev_net(dev));
943 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200944 ret = __dev_alloc_name(net, name, buf);
945 if (ret >= 0)
946 strlcpy(dev->name, buf, IFNAMSIZ);
947 return ret;
948}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700949EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200950
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000951static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt)
Octavian Purdilad9031022009-11-18 02:36:59 +0000952{
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000953 struct net *net;
954
955 BUG_ON(!dev_net(dev));
956 net = dev_net(dev);
957
Octavian Purdilad9031022009-11-18 02:36:59 +0000958 if (!dev_valid_name(name))
959 return -EINVAL;
960
961 if (fmt && strchr(name, '%'))
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000962 return dev_alloc_name(dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +0000963 else if (__dev_get_by_name(net, name))
964 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000965 else if (dev->name != name)
966 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +0000967
968 return 0;
969}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970
971/**
972 * dev_change_name - change name of a device
973 * @dev: device
974 * @newname: name (or format string) must be at least IFNAMSIZ
975 *
976 * Change name of a device, can pass format strings "eth%d".
977 * for wildcarding.
978 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -0700979int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980{
Herbert Xufcc5a032007-07-30 17:03:38 -0700981 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -0700983 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -0700984 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985
986 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900987 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900989 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990 if (dev->flags & IFF_UP)
991 return -EBUSY;
992
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -0700993 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
994 return 0;
995
Herbert Xufcc5a032007-07-30 17:03:38 -0700996 memcpy(oldname, dev->name, IFNAMSIZ);
997
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000998 err = dev_get_valid_name(dev, newname, 1);
Octavian Purdilad9031022009-11-18 02:36:59 +0000999 if (err < 0)
1000 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001
Herbert Xufcc5a032007-07-30 17:03:38 -07001002rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001003 ret = device_rename(&dev->dev, dev->name);
1004 if (ret) {
1005 memcpy(dev->name, oldname, IFNAMSIZ);
1006 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001007 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001008
1009 write_lock_bh(&dev_base_lock);
Eric W. Biederman92749822007-04-03 00:07:30 -06001010 hlist_del(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001011 write_unlock_bh(&dev_base_lock);
1012
1013 synchronize_rcu();
1014
1015 write_lock_bh(&dev_base_lock);
1016 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001017 write_unlock_bh(&dev_base_lock);
1018
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001019 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001020 ret = notifier_to_errno(ret);
1021
1022 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001023 /* err >= 0 after dev_alloc_name() or stores the first errno */
1024 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001025 err = ret;
1026 memcpy(dev->name, oldname, IFNAMSIZ);
1027 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001028 } else {
1029 printk(KERN_ERR
1030 "%s: name change rollback failed: %d.\n",
1031 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001032 }
1033 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034
1035 return err;
1036}
1037
1038/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001039 * dev_set_alias - change ifalias of a device
1040 * @dev: device
1041 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001042 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001043 *
1044 * Set ifalias for a device,
1045 */
1046int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1047{
1048 ASSERT_RTNL();
1049
1050 if (len >= IFALIASZ)
1051 return -EINVAL;
1052
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001053 if (!len) {
1054 if (dev->ifalias) {
1055 kfree(dev->ifalias);
1056 dev->ifalias = NULL;
1057 }
1058 return 0;
1059 }
1060
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001061 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001062 if (!dev->ifalias)
1063 return -ENOMEM;
1064
1065 strlcpy(dev->ifalias, alias, len+1);
1066 return len;
1067}
1068
1069
1070/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001071 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001072 * @dev: device to cause notification
1073 *
1074 * Called to indicate a device has changed features.
1075 */
1076void netdev_features_change(struct net_device *dev)
1077{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001078 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001079}
1080EXPORT_SYMBOL(netdev_features_change);
1081
1082/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 * netdev_state_change - device changes state
1084 * @dev: device to cause notification
1085 *
1086 * Called to indicate a device has changed state. This function calls
1087 * the notifier chains for netdev_chain and sends a NEWLINK message
1088 * to the routing socket.
1089 */
1090void netdev_state_change(struct net_device *dev)
1091{
1092 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001093 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1095 }
1096}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001097EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001099int netdev_bonding_change(struct net_device *dev, unsigned long event)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001100{
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001101 return call_netdevice_notifiers(event, dev);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001102}
1103EXPORT_SYMBOL(netdev_bonding_change);
1104
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105/**
1106 * dev_load - load a network module
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001107 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 * @name: name of interface
1109 *
1110 * If a network interface is not present and the process has suitable
1111 * privileges this function loads the module. If module loading is not
1112 * available in this kernel then it becomes a nop.
1113 */
1114
Eric W. Biederman881d9662007-09-17 11:56:21 -07001115void dev_load(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001116{
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001117 struct net_device *dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118
Eric Dumazet72c95282009-10-30 07:11:27 +00001119 rcu_read_lock();
1120 dev = dev_get_by_name_rcu(net, name);
1121 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122
Eric Parisa8f80e82009-08-13 09:44:51 -04001123 if (!dev && capable(CAP_NET_ADMIN))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124 request_module("%s", name);
1125}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001126EXPORT_SYMBOL(dev_load);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127
Patrick McHardybd380812010-02-26 06:34:53 +00001128static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001130 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001131 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001133 ASSERT_RTNL();
1134
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136 * Is it even present?
1137 */
1138 if (!netif_device_present(dev))
1139 return -ENODEV;
1140
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001141 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1142 ret = notifier_to_errno(ret);
1143 if (ret)
1144 return ret;
1145
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146 /*
1147 * Call device private open method
1148 */
1149 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001150
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001151 if (ops->ndo_validate_addr)
1152 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001153
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001154 if (!ret && ops->ndo_open)
1155 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001157 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001158 * If it went open OK then:
1159 */
1160
Jeff Garzikbada3392007-10-23 20:19:37 -07001161 if (ret)
1162 clear_bit(__LINK_STATE_START, &dev->state);
1163 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164 /*
1165 * Set the flags.
1166 */
1167 dev->flags |= IFF_UP;
1168
1169 /*
Dan Williams649274d2009-01-11 00:20:39 -08001170 * Enable NET_DMA
1171 */
David S. Millerb4bd07c2009-02-06 22:06:43 -08001172 net_dmaengine_get();
Dan Williams649274d2009-01-11 00:20:39 -08001173
1174 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 * Initialize multicasting status
1176 */
Patrick McHardy4417da62007-06-27 01:28:10 -07001177 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178
1179 /*
1180 * Wakeup transmit queue engine
1181 */
1182 dev_activate(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001183 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001184
Linus Torvalds1da177e2005-04-16 15:20:36 -07001185 return ret;
1186}
Patrick McHardybd380812010-02-26 06:34:53 +00001187
1188/**
1189 * dev_open - prepare an interface for use.
1190 * @dev: device to open
1191 *
1192 * Takes a device from down to up state. The device's private open
1193 * function is invoked and then the multicast lists are loaded. Finally
1194 * the device is moved into the up state and a %NETDEV_UP message is
1195 * sent to the netdev notifier chain.
1196 *
1197 * Calling this function on an active interface is a nop. On a failure
1198 * a negative errno code is returned.
1199 */
1200int dev_open(struct net_device *dev)
1201{
1202 int ret;
1203
1204 /*
1205 * Is it already up?
1206 */
1207 if (dev->flags & IFF_UP)
1208 return 0;
1209
1210 /*
1211 * Open device
1212 */
1213 ret = __dev_open(dev);
1214 if (ret < 0)
1215 return ret;
1216
1217 /*
1218 * ... and announce new interface.
1219 */
1220 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1221 call_netdevice_notifiers(NETDEV_UP, dev);
1222
1223 return ret;
1224}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001225EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226
Octavian Purdila44345722010-12-13 12:44:07 +00001227static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228{
Octavian Purdila44345722010-12-13 12:44:07 +00001229 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001230
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001231 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001232 might_sleep();
1233
Octavian Purdila44345722010-12-13 12:44:07 +00001234 list_for_each_entry(dev, head, unreg_list) {
1235 /*
1236 * Tell people we are going down, so that they can
1237 * prepare to death, when device is still operating.
1238 */
1239 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240
Octavian Purdila44345722010-12-13 12:44:07 +00001241 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242
Octavian Purdila44345722010-12-13 12:44:07 +00001243 /* Synchronize to scheduled poll. We cannot touch poll list, it
1244 * can be even on different cpu. So just clear netif_running().
1245 *
1246 * dev->stop() will invoke napi_disable() on all of it's
1247 * napi_struct instances on this device.
1248 */
1249 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1250 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251
Octavian Purdila44345722010-12-13 12:44:07 +00001252 dev_deactivate_many(head);
1253
1254 list_for_each_entry(dev, head, unreg_list) {
1255 const struct net_device_ops *ops = dev->netdev_ops;
1256
1257 /*
1258 * Call the device specific close. This cannot fail.
1259 * Only if device is UP
1260 *
1261 * We allow it to be called even after a DETACH hot-plug
1262 * event.
1263 */
1264 if (ops->ndo_stop)
1265 ops->ndo_stop(dev);
1266
1267 /*
1268 * Device is now down.
1269 */
1270
1271 dev->flags &= ~IFF_UP;
1272
1273 /*
1274 * Shutdown NET_DMA
1275 */
1276 net_dmaengine_put();
1277 }
1278
1279 return 0;
1280}
1281
1282static int __dev_close(struct net_device *dev)
1283{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001284 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001285 LIST_HEAD(single);
1286
1287 list_add(&dev->unreg_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001288 retval = __dev_close_many(&single);
1289 list_del(&single);
1290 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001291}
1292
Eric Dumazet3fbd8752011-01-19 21:23:22 +00001293static int dev_close_many(struct list_head *head)
Octavian Purdila44345722010-12-13 12:44:07 +00001294{
1295 struct net_device *dev, *tmp;
1296 LIST_HEAD(tmp_list);
1297
1298 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1299 if (!(dev->flags & IFF_UP))
1300 list_move(&dev->unreg_list, &tmp_list);
1301
1302 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001303
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304 /*
Octavian Purdila44345722010-12-13 12:44:07 +00001305 * Tell people we are down
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306 */
Octavian Purdila44345722010-12-13 12:44:07 +00001307 list_for_each_entry(dev, head, unreg_list) {
1308 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1309 call_netdevice_notifiers(NETDEV_DOWN, dev);
1310 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311
Octavian Purdila44345722010-12-13 12:44:07 +00001312 /* rollback_registered_many needs the complete original list */
1313 list_splice(&tmp_list, head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314 return 0;
1315}
Patrick McHardybd380812010-02-26 06:34:53 +00001316
1317/**
1318 * dev_close - shutdown an interface.
1319 * @dev: device to shutdown
1320 *
1321 * This function moves an active device into down state. A
1322 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1323 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1324 * chain.
1325 */
1326int dev_close(struct net_device *dev)
1327{
Octavian Purdila44345722010-12-13 12:44:07 +00001328 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001329
Octavian Purdila44345722010-12-13 12:44:07 +00001330 list_add(&dev->unreg_list, &single);
1331 dev_close_many(&single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001332 list_del(&single);
Patrick McHardybd380812010-02-26 06:34:53 +00001333 return 0;
1334}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001335EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336
1337
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001338/**
1339 * dev_disable_lro - disable Large Receive Offload on a device
1340 * @dev: device
1341 *
1342 * Disable Large Receive Offload (LRO) on a net device. Must be
1343 * called under RTNL. This is needed if received packets may be
1344 * forwarded to another interface.
1345 */
1346void dev_disable_lro(struct net_device *dev)
1347{
1348 if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&
1349 dev->ethtool_ops->set_flags) {
1350 u32 flags = dev->ethtool_ops->get_flags(dev);
1351 if (flags & ETH_FLAG_LRO) {
1352 flags &= ~ETH_FLAG_LRO;
1353 dev->ethtool_ops->set_flags(dev, flags);
1354 }
1355 }
1356 WARN_ON(dev->features & NETIF_F_LRO);
1357}
1358EXPORT_SYMBOL(dev_disable_lro);
1359
1360
Eric W. Biederman881d9662007-09-17 11:56:21 -07001361static int dev_boot_phase = 1;
1362
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363/*
1364 * Device change register/unregister. These are not inline or static
1365 * as we export them to the world.
1366 */
1367
1368/**
1369 * register_netdevice_notifier - register a network notifier block
1370 * @nb: notifier
1371 *
1372 * Register a notifier to be called when network device events occur.
1373 * The notifier passed is linked into the kernel structures and must
1374 * not be reused until it has been unregistered. A negative errno code
1375 * is returned on a failure.
1376 *
1377 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001378 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379 * view of the network device list.
1380 */
1381
1382int register_netdevice_notifier(struct notifier_block *nb)
1383{
1384 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001385 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001386 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387 int err;
1388
1389 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001390 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001391 if (err)
1392 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001393 if (dev_boot_phase)
1394 goto unlock;
1395 for_each_net(net) {
1396 for_each_netdev(net, dev) {
1397 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1398 err = notifier_to_errno(err);
1399 if (err)
1400 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401
Eric W. Biederman881d9662007-09-17 11:56:21 -07001402 if (!(dev->flags & IFF_UP))
1403 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001404
Eric W. Biederman881d9662007-09-17 11:56:21 -07001405 nb->notifier_call(nb, NETDEV_UP, dev);
1406 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001408
1409unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410 rtnl_unlock();
1411 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001412
1413rollback:
1414 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001415 for_each_net(net) {
1416 for_each_netdev(net, dev) {
1417 if (dev == last)
1418 break;
Herbert Xufcc5a032007-07-30 17:03:38 -07001419
Eric W. Biederman881d9662007-09-17 11:56:21 -07001420 if (dev->flags & IFF_UP) {
1421 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1422 nb->notifier_call(nb, NETDEV_DOWN, dev);
1423 }
1424 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00001425 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001426 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001427 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001428
1429 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001430 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001432EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001433
1434/**
1435 * unregister_netdevice_notifier - unregister a network notifier block
1436 * @nb: notifier
1437 *
1438 * Unregister a notifier previously registered by
1439 * register_netdevice_notifier(). The notifier is unlinked into the
1440 * kernel structures and may then be reused. A negative errno code
1441 * is returned on a failure.
1442 */
1443
1444int unregister_netdevice_notifier(struct notifier_block *nb)
1445{
Herbert Xu9f514952006-03-25 01:24:25 -08001446 int err;
1447
1448 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001449 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xu9f514952006-03-25 01:24:25 -08001450 rtnl_unlock();
1451 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001453EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454
1455/**
1456 * call_netdevice_notifiers - call all network notifier blocks
1457 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001458 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001459 *
1460 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001461 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462 */
1463
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001464int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465{
Jiri Pirkoab930472010-04-20 01:45:37 -07001466 ASSERT_RTNL();
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001467 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468}
1469
1470/* When > 0 there are consumers of rx skb time stamps */
1471static atomic_t netstamp_needed = ATOMIC_INIT(0);
1472
1473void net_enable_timestamp(void)
1474{
1475 atomic_inc(&netstamp_needed);
1476}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001477EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478
1479void net_disable_timestamp(void)
1480{
1481 atomic_dec(&netstamp_needed);
1482}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001483EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484
Eric Dumazet3b098e22010-05-15 23:57:10 -07001485static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001486{
1487 if (atomic_read(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001488 __net_timestamp(skb);
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07001489 else
1490 skb->tstamp.tv64 = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491}
1492
Eric Dumazet3b098e22010-05-15 23:57:10 -07001493static inline void net_timestamp_check(struct sk_buff *skb)
1494{
1495 if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))
1496 __net_timestamp(skb);
1497}
1498
Arnd Bergmann44540962009-11-26 06:07:08 +00001499/**
1500 * dev_forward_skb - loopback an skb to another netif
1501 *
1502 * @dev: destination network device
1503 * @skb: buffer to forward
1504 *
1505 * return values:
1506 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001507 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001508 *
1509 * dev_forward_skb can be used for injecting an skb from the
1510 * start_xmit function of one device into the receive queue
1511 * of another device.
1512 *
1513 * The receiving device may be in another namespace, so
1514 * we have to clear all information in the skb that could
1515 * impact namespace isolation.
1516 */
1517int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1518{
1519 skb_orphan(skb);
Ben Greearc736eef2010-07-22 09:54:47 +00001520 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001521
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001522 if (unlikely(!(dev->flags & IFF_UP) ||
David S. Miller2198a102010-10-21 08:43:05 -07001523 (skb->len > (dev->mtu + dev->hard_header_len + VLAN_HLEN)))) {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001524 atomic_long_inc(&dev->rx_dropped);
Eric Dumazet6ec82562010-05-06 00:53:53 -07001525 kfree_skb(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001526 return NET_RX_DROP;
Eric Dumazet6ec82562010-05-06 00:53:53 -07001527 }
Arnd Bergmann8a83a002010-01-30 12:23:03 +00001528 skb_set_dev(skb, dev);
Arnd Bergmann44540962009-11-26 06:07:08 +00001529 skb->tstamp.tv64 = 0;
1530 skb->pkt_type = PACKET_HOST;
1531 skb->protocol = eth_type_trans(skb, dev);
Arnd Bergmann44540962009-11-26 06:07:08 +00001532 return netif_rx(skb);
1533}
1534EXPORT_SYMBOL_GPL(dev_forward_skb);
1535
Changli Gao71d9dec2010-12-15 19:57:25 +00001536static inline int deliver_skb(struct sk_buff *skb,
1537 struct packet_type *pt_prev,
1538 struct net_device *orig_dev)
1539{
1540 atomic_inc(&skb->users);
1541 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1542}
1543
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544/*
1545 * Support routine. Sends outgoing frames to any network
1546 * taps currently in use.
1547 */
1548
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001549static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550{
1551 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001552 struct sk_buff *skb2 = NULL;
1553 struct packet_type *pt_prev = NULL;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001554
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555 rcu_read_lock();
1556 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1557 /* Never send packets back to the socket
1558 * they originated from - MvS (miquels@drinkel.ow.org)
1559 */
1560 if ((ptype->dev == dev || !ptype->dev) &&
1561 (ptype->af_packet_priv == NULL ||
1562 (struct sock *)ptype->af_packet_priv != skb->sk)) {
Changli Gao71d9dec2010-12-15 19:57:25 +00001563 if (pt_prev) {
1564 deliver_skb(skb2, pt_prev, skb->dev);
1565 pt_prev = ptype;
1566 continue;
1567 }
1568
1569 skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001570 if (!skb2)
1571 break;
1572
Eric Dumazet70978182010-12-20 21:22:51 +00001573 net_timestamp_set(skb2);
1574
Linus Torvalds1da177e2005-04-16 15:20:36 -07001575 /* skb->nh should be correctly
1576 set by sender, so that the second statement is
1577 just protection against buggy protocols.
1578 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001579 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001581 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001582 skb2->network_header > skb2->tail) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 if (net_ratelimit())
1584 printk(KERN_CRIT "protocol %04x is "
1585 "buggy, dev %s\n",
Sebastian Andrzej Siewior70777d02010-06-30 10:39:19 -07001586 ntohs(skb2->protocol),
1587 dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001588 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589 }
1590
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001591 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592 skb2->pkt_type = PACKET_OUTGOING;
Changli Gao71d9dec2010-12-15 19:57:25 +00001593 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594 }
1595 }
Changli Gao71d9dec2010-12-15 19:57:25 +00001596 if (pt_prev)
1597 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598 rcu_read_unlock();
1599}
1600
John Fastabend4f57c082011-01-17 08:06:04 +00001601/* netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1602 * @dev: Network device
1603 * @txq: number of queues available
1604 *
1605 * If real_num_tx_queues is changed the tc mappings may no longer be
1606 * valid. To resolve this verify the tc mapping remains valid and if
1607 * not NULL the mapping. With no priorities mapping to this
1608 * offset/count pair it will no longer be used. In the worst case TC0
1609 * is invalid nothing can be done so disable priority mappings. If is
1610 * expected that drivers will fix this mapping if they can before
1611 * calling netif_set_real_num_tx_queues.
1612 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001613static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001614{
1615 int i;
1616 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1617
1618 /* If TC0 is invalidated disable TC mapping */
1619 if (tc->offset + tc->count > txq) {
1620 pr_warning("Number of in use tx queues changed "
1621 "invalidating tc mappings. Priority "
1622 "traffic classification disabled!\n");
1623 dev->num_tc = 0;
1624 return;
1625 }
1626
1627 /* Invalidated prio to tc mappings set to TC0 */
1628 for (i = 1; i < TC_BITMASK + 1; i++) {
1629 int q = netdev_get_prio_tc_map(dev, i);
1630
1631 tc = &dev->tc_to_txq[q];
1632 if (tc->offset + tc->count > txq) {
1633 pr_warning("Number of in use tx queues "
1634 "changed. Priority %i to tc "
1635 "mapping %i is no longer valid "
1636 "setting map to 0\n",
1637 i, q);
1638 netdev_set_prio_tc_map(dev, i, 0);
1639 }
1640 }
1641}
1642
John Fastabendf0796d52010-07-01 13:21:57 +00001643/*
1644 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1645 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1646 */
Tom Herberte6484932010-10-18 18:04:39 +00001647int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00001648{
Tom Herbert1d24eb42010-11-21 13:17:27 +00001649 int rc;
1650
Tom Herberte6484932010-10-18 18:04:39 +00001651 if (txq < 1 || txq > dev->num_tx_queues)
1652 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00001653
Ben Hutchings5c565802011-02-15 19:39:21 +00001654 if (dev->reg_state == NETREG_REGISTERED ||
1655 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00001656 ASSERT_RTNL();
1657
Tom Herbert1d24eb42010-11-21 13:17:27 +00001658 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1659 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00001660 if (rc)
1661 return rc;
1662
John Fastabend4f57c082011-01-17 08:06:04 +00001663 if (dev->num_tc)
1664 netif_setup_tc(dev, txq);
1665
Tom Herberte6484932010-10-18 18:04:39 +00001666 if (txq < dev->real_num_tx_queues)
1667 qdisc_reset_all_tx_gt(dev, txq);
John Fastabendf0796d52010-07-01 13:21:57 +00001668 }
Tom Herberte6484932010-10-18 18:04:39 +00001669
1670 dev->real_num_tx_queues = txq;
1671 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00001672}
1673EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08001674
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001675#ifdef CONFIG_RPS
1676/**
1677 * netif_set_real_num_rx_queues - set actual number of RX queues used
1678 * @dev: Network device
1679 * @rxq: Actual number of RX queues
1680 *
1681 * This must be called either with the rtnl_lock held or before
1682 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07001683 * negative error code. If called before registration, it always
1684 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001685 */
1686int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1687{
1688 int rc;
1689
Tom Herbertbd25fa72010-10-18 18:00:16 +00001690 if (rxq < 1 || rxq > dev->num_rx_queues)
1691 return -EINVAL;
1692
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001693 if (dev->reg_state == NETREG_REGISTERED) {
1694 ASSERT_RTNL();
1695
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001696 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1697 rxq);
1698 if (rc)
1699 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001700 }
1701
1702 dev->real_num_rx_queues = rxq;
1703 return 0;
1704}
1705EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1706#endif
1707
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001708static inline void __netif_reschedule(struct Qdisc *q)
1709{
1710 struct softnet_data *sd;
1711 unsigned long flags;
1712
1713 local_irq_save(flags);
1714 sd = &__get_cpu_var(softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00001715 q->next_sched = NULL;
1716 *sd->output_queue_tailp = q;
1717 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001718 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1719 local_irq_restore(flags);
1720}
1721
David S. Miller37437bb2008-07-16 02:15:04 -07001722void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08001723{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001724 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1725 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08001726}
1727EXPORT_SYMBOL(__netif_schedule);
1728
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001729void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08001730{
David S. Miller3578b0c2010-08-03 00:24:04 -07001731 if (atomic_dec_and_test(&skb->users)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001732 struct softnet_data *sd;
1733 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08001734
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001735 local_irq_save(flags);
1736 sd = &__get_cpu_var(softnet_data);
1737 skb->next = sd->completion_queue;
1738 sd->completion_queue = skb;
1739 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1740 local_irq_restore(flags);
1741 }
Denis Vlasenko56079432006-03-29 15:57:29 -08001742}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001743EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08001744
1745void dev_kfree_skb_any(struct sk_buff *skb)
1746{
1747 if (in_irq() || irqs_disabled())
1748 dev_kfree_skb_irq(skb);
1749 else
1750 dev_kfree_skb(skb);
1751}
1752EXPORT_SYMBOL(dev_kfree_skb_any);
1753
1754
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001755/**
1756 * netif_device_detach - mark device as removed
1757 * @dev: network device
1758 *
1759 * Mark device as removed from system and therefore no longer available.
1760 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001761void netif_device_detach(struct net_device *dev)
1762{
1763 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1764 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001765 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001766 }
1767}
1768EXPORT_SYMBOL(netif_device_detach);
1769
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001770/**
1771 * netif_device_attach - mark device as attached
1772 * @dev: network device
1773 *
1774 * Mark device as attached from system and restart if needed.
1775 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001776void netif_device_attach(struct net_device *dev)
1777{
1778 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1779 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001780 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001781 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001782 }
1783}
1784EXPORT_SYMBOL(netif_device_attach);
1785
Arnd Bergmann8a83a002010-01-30 12:23:03 +00001786/**
1787 * skb_dev_set -- assign a new device to a buffer
1788 * @skb: buffer for the new device
1789 * @dev: network device
1790 *
1791 * If an skb is owned by a device already, we have to reset
1792 * all data private to the namespace a device belongs to
1793 * before assigning it a new device.
1794 */
1795#ifdef CONFIG_NET_NS
1796void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
1797{
1798 skb_dst_drop(skb);
1799 if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
1800 secpath_reset(skb);
1801 nf_reset(skb);
1802 skb_init_secmark(skb);
1803 skb->mark = 0;
1804 skb->priority = 0;
1805 skb->nf_trace = 0;
1806 skb->ipvs_property = 0;
1807#ifdef CONFIG_NET_SCHED
1808 skb->tc_index = 0;
1809#endif
1810 }
1811 skb->dev = dev;
1812}
1813EXPORT_SYMBOL(skb_set_dev);
1814#endif /* CONFIG_NET_NS */
1815
Linus Torvalds1da177e2005-04-16 15:20:36 -07001816/*
1817 * Invalidate hardware checksum when packet is to be mangled, and
1818 * complete checksum manually on outgoing path.
1819 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07001820int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001821{
Al Virod3bc23e2006-11-14 21:24:49 -08001822 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07001823 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001824
Patrick McHardy84fa7932006-08-29 16:44:56 -07001825 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07001826 goto out_set_summed;
1827
1828 if (unlikely(skb_shinfo(skb)->gso_size)) {
Herbert Xua430a432006-07-08 13:34:56 -07001829 /* Let GSO fix up the checksum. */
1830 goto out_set_summed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001831 }
1832
Michał Mirosław55508d62010-12-14 15:24:08 +00001833 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07001834 BUG_ON(offset >= skb_headlen(skb));
1835 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1836
1837 offset += skb->csum_offset;
1838 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1839
1840 if (skb_cloned(skb) &&
1841 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001842 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1843 if (ret)
1844 goto out;
1845 }
1846
Herbert Xua0308472007-10-15 01:47:15 -07001847 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07001848out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001849 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001850out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001851 return ret;
1852}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001853EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001855/**
1856 * skb_gso_segment - Perform segmentation on skb.
1857 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07001858 * @features: features for the output path (see dev->features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001859 *
1860 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07001861 *
1862 * It may return NULL if the skb requires no segmentation. This is
1863 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001864 */
Michał Mirosław04ed3e72011-01-24 15:32:47 -08001865struct sk_buff *skb_gso_segment(struct sk_buff *skb, u32 features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001866{
1867 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1868 struct packet_type *ptype;
Al Viro252e33462006-11-14 20:48:11 -08001869 __be16 type = skb->protocol;
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001870 int vlan_depth = ETH_HLEN;
Herbert Xua430a432006-07-08 13:34:56 -07001871 int err;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001872
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001873 while (type == htons(ETH_P_8021Q)) {
1874 struct vlan_hdr *vh;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001875
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001876 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
Jesse Gross7b9c6092010-10-20 13:56:04 +00001877 return ERR_PTR(-EINVAL);
1878
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001879 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
1880 type = vh->h_vlan_encapsulated_proto;
1881 vlan_depth += VLAN_HLEN;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001882 }
1883
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001884 skb_reset_mac_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001885 skb->mac_len = skb->network_header - skb->mac_header;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001886 __skb_pull(skb, skb->mac_len);
1887
Herbert Xu67fd1a72009-01-19 16:26:44 -08001888 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1889 struct net_device *dev = skb->dev;
1890 struct ethtool_drvinfo info = {};
1891
1892 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1893 dev->ethtool_ops->get_drvinfo(dev, &info);
1894
Joe Perchesb194a362010-10-30 11:08:52 +00001895 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d ip_summed=%d\n",
Herbert Xu67fd1a72009-01-19 16:26:44 -08001896 info.driver, dev ? dev->features : 0L,
1897 skb->sk ? skb->sk->sk_route_caps : 0L,
1898 skb->len, skb->data_len, skb->ip_summed);
1899
Herbert Xua430a432006-07-08 13:34:56 -07001900 if (skb_header_cloned(skb) &&
1901 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1902 return ERR_PTR(err);
1903 }
1904
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001905 rcu_read_lock();
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08001906 list_for_each_entry_rcu(ptype,
1907 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001908 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07001909 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Herbert Xua430a432006-07-08 13:34:56 -07001910 err = ptype->gso_send_check(skb);
1911 segs = ERR_PTR(err);
1912 if (err || skb_gso_ok(skb, features))
1913 break;
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001914 __skb_push(skb, (skb->data -
1915 skb_network_header(skb)));
Herbert Xua430a432006-07-08 13:34:56 -07001916 }
Herbert Xu576a30e2006-06-27 13:22:38 -07001917 segs = ptype->gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001918 break;
1919 }
1920 }
1921 rcu_read_unlock();
1922
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001923 __skb_push(skb, skb->data - skb_mac_header(skb));
Herbert Xu576a30e2006-06-27 13:22:38 -07001924
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001925 return segs;
1926}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001927EXPORT_SYMBOL(skb_gso_segment);
1928
Herbert Xufb286bb2005-11-10 13:01:24 -08001929/* Take action when hardware reception checksum errors are detected. */
1930#ifdef CONFIG_BUG
1931void netdev_rx_csum_fault(struct net_device *dev)
1932{
1933 if (net_ratelimit()) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001934 printk(KERN_ERR "%s: hw csum failure.\n",
Stephen Hemminger246a4212005-12-08 15:21:39 -08001935 dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08001936 dump_stack();
1937 }
1938}
1939EXPORT_SYMBOL(netdev_rx_csum_fault);
1940#endif
1941
Linus Torvalds1da177e2005-04-16 15:20:36 -07001942/* Actually, we should eliminate this check as soon as we know, that:
1943 * 1. IOMMU is present and allows to map all the memory.
1944 * 2. No high memory really exists on this machine.
1945 */
1946
Eric Dumazet9092c652010-04-02 13:34:49 -07001947static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001948{
Herbert Xu3d3a8532006-06-27 13:33:10 -07001949#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07001950 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00001951 if (!(dev->features & NETIF_F_HIGHDMA)) {
1952 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1953 if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1954 return 1;
1955 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001956
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00001957 if (PCI_DMA_BUS_IS_PHYS) {
1958 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001959
Eric Dumazet9092c652010-04-02 13:34:49 -07001960 if (!pdev)
1961 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00001962 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1963 dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);
1964 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
1965 return 1;
1966 }
1967 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07001968#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001969 return 0;
1970}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001971
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001972struct dev_gso_cb {
1973 void (*destructor)(struct sk_buff *skb);
1974};
1975
1976#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1977
1978static void dev_gso_skb_destructor(struct sk_buff *skb)
1979{
1980 struct dev_gso_cb *cb;
1981
1982 do {
1983 struct sk_buff *nskb = skb->next;
1984
1985 skb->next = nskb->next;
1986 nskb->next = NULL;
1987 kfree_skb(nskb);
1988 } while (skb->next);
1989
1990 cb = DEV_GSO_CB(skb);
1991 if (cb->destructor)
1992 cb->destructor(skb);
1993}
1994
1995/**
1996 * dev_gso_segment - Perform emulated hardware segmentation on skb.
1997 * @skb: buffer to segment
Jesse Gross91ecb632011-01-09 06:23:33 +00001998 * @features: device features as applicable to this skb
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001999 *
2000 * This function segments the given skb and stores the list of segments
2001 * in skb->next.
2002 */
Jesse Gross91ecb632011-01-09 06:23:33 +00002003static int dev_gso_segment(struct sk_buff *skb, int features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002004{
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002005 struct sk_buff *segs;
2006
Herbert Xu576a30e2006-06-27 13:22:38 -07002007 segs = skb_gso_segment(skb, features);
2008
2009 /* Verifying header integrity only. */
2010 if (!segs)
2011 return 0;
2012
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07002013 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002014 return PTR_ERR(segs);
2015
2016 skb->next = segs;
2017 DEV_GSO_CB(skb)->destructor = skb->destructor;
2018 skb->destructor = dev_gso_skb_destructor;
2019
2020 return 0;
2021}
2022
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002023/*
2024 * Try to orphan skb early, right before transmission by the device.
Oliver Hartkopp2244d072010-08-17 08:59:14 +00002025 * We cannot orphan skb if tx timestamp is requested or the sk-reference
2026 * is needed on driver level for other reasons, e.g. see net/can/raw.c
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002027 */
2028static inline void skb_orphan_try(struct sk_buff *skb)
2029{
Eric Dumazet87fd3082010-07-13 05:24:20 +00002030 struct sock *sk = skb->sk;
2031
Oliver Hartkopp2244d072010-08-17 08:59:14 +00002032 if (sk && !skb_shinfo(skb)->tx_flags) {
Eric Dumazet87fd3082010-07-13 05:24:20 +00002033 /* skb_tx_hash() wont be able to get sk.
2034 * We copy sk_hash into skb->rxhash
2035 */
2036 if (!skb->rxhash)
2037 skb->rxhash = sk->sk_hash;
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002038 skb_orphan(skb);
Eric Dumazet87fd3082010-07-13 05:24:20 +00002039 }
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002040}
2041
Jesse Gross03634662011-01-09 06:23:35 +00002042static bool can_checksum_protocol(unsigned long features, __be16 protocol)
2043{
2044 return ((features & NETIF_F_GEN_CSUM) ||
2045 ((features & NETIF_F_V4_CSUM) &&
2046 protocol == htons(ETH_P_IP)) ||
2047 ((features & NETIF_F_V6_CSUM) &&
2048 protocol == htons(ETH_P_IPV6)) ||
2049 ((features & NETIF_F_FCOE_CRC) &&
2050 protocol == htons(ETH_P_FCOE)));
2051}
2052
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002053static u32 harmonize_features(struct sk_buff *skb, __be16 protocol, u32 features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002054{
Eric Dumazetd4027862011-01-19 00:51:36 +00002055 if (!can_checksum_protocol(features, protocol)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002056 features &= ~NETIF_F_ALL_CSUM;
2057 features &= ~NETIF_F_SG;
2058 } else if (illegal_highdma(skb->dev, skb)) {
2059 features &= ~NETIF_F_SG;
2060 }
2061
2062 return features;
2063}
2064
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002065u32 netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002066{
2067 __be16 protocol = skb->protocol;
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002068 u32 features = skb->dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002069
2070 if (protocol == htons(ETH_P_8021Q)) {
2071 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2072 protocol = veh->h_vlan_encapsulated_proto;
Jesse Grossf01a5232011-01-09 06:23:31 +00002073 } else if (!vlan_tx_tag_present(skb)) {
2074 return harmonize_features(skb, protocol, features);
2075 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002076
Jesse Gross6ee400a2011-01-17 20:46:00 +00002077 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
Jesse Grossf01a5232011-01-09 06:23:31 +00002078
2079 if (protocol != htons(ETH_P_8021Q)) {
2080 return harmonize_features(skb, protocol, features);
2081 } else {
2082 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
Jesse Gross6ee400a2011-01-17 20:46:00 +00002083 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
Jesse Grossf01a5232011-01-09 06:23:31 +00002084 return harmonize_features(skb, protocol, features);
2085 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002086}
Jesse Grossf01a5232011-01-09 06:23:31 +00002087EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002088
John Fastabend6afff0c2010-06-16 14:18:12 +00002089/*
2090 * Returns true if either:
2091 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2092 * 2. skb is fragmented and the device does not support SG, or if
2093 * at least one of fragments is in highmem and device does not
2094 * support DMA from it.
2095 */
2096static inline int skb_needs_linearize(struct sk_buff *skb,
Jesse Gross02932ce2011-01-09 06:23:34 +00002097 int features)
John Fastabend6afff0c2010-06-16 14:18:12 +00002098{
Jesse Gross02932ce2011-01-09 06:23:34 +00002099 return skb_is_nonlinear(skb) &&
2100 ((skb_has_frag_list(skb) &&
2101 !(features & NETIF_F_FRAGLIST)) ||
Jesse Grosse1e78db2010-10-29 12:14:53 +00002102 (skb_shinfo(skb)->nr_frags &&
Jesse Gross02932ce2011-01-09 06:23:34 +00002103 !(features & NETIF_F_SG)));
John Fastabend6afff0c2010-06-16 14:18:12 +00002104}
2105
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002106int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2107 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002108{
Stephen Hemminger00829822008-11-20 20:14:53 -08002109 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00002110 int rc = NETDEV_TX_OK;
Stephen Hemminger00829822008-11-20 20:14:53 -08002111
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002112 if (likely(!skb->next)) {
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002113 u32 features;
Jesse Grossfc741212011-01-09 06:23:32 +00002114
Eric Dumazet93f154b2009-05-18 22:19:19 -07002115 /*
2116 * If device doesnt need skb->dst, release it right now while
2117 * its hot in this cpu cache
2118 */
Eric Dumazetadf30902009-06-02 05:19:30 +00002119 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2120 skb_dst_drop(skb);
2121
Eric Dumazet15c2d75f2010-12-07 00:30:37 +00002122 if (!list_empty(&ptype_all))
2123 dev_queue_xmit_nit(skb, dev);
2124
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002125 skb_orphan_try(skb);
David S. Miller9ccb8972010-04-22 01:02:07 -07002126
Jesse Grossfc741212011-01-09 06:23:32 +00002127 features = netif_skb_features(skb);
2128
Jesse Gross7b9c6092010-10-20 13:56:04 +00002129 if (vlan_tx_tag_present(skb) &&
Jesse Grossfc741212011-01-09 06:23:32 +00002130 !(features & NETIF_F_HW_VLAN_TX)) {
Jesse Gross7b9c6092010-10-20 13:56:04 +00002131 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2132 if (unlikely(!skb))
2133 goto out;
2134
2135 skb->vlan_tci = 0;
2136 }
2137
Jesse Grossfc741212011-01-09 06:23:32 +00002138 if (netif_needs_gso(skb, features)) {
Jesse Gross91ecb632011-01-09 06:23:33 +00002139 if (unlikely(dev_gso_segment(skb, features)))
David S. Miller9ccb8972010-04-22 01:02:07 -07002140 goto out_kfree_skb;
2141 if (skb->next)
2142 goto gso;
John Fastabend6afff0c2010-06-16 14:18:12 +00002143 } else {
Jesse Gross02932ce2011-01-09 06:23:34 +00002144 if (skb_needs_linearize(skb, features) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002145 __skb_linearize(skb))
2146 goto out_kfree_skb;
2147
2148 /* If packet is not checksummed and device does not
2149 * support checksumming for this protocol, complete
2150 * checksumming here.
2151 */
2152 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Michał Mirosław55508d62010-12-14 15:24:08 +00002153 skb_set_transport_header(skb,
2154 skb_checksum_start_offset(skb));
Jesse Gross03634662011-01-09 06:23:35 +00002155 if (!(features & NETIF_F_ALL_CSUM) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002156 skb_checksum_help(skb))
2157 goto out_kfree_skb;
2158 }
David S. Miller9ccb8972010-04-22 01:02:07 -07002159 }
2160
Patrick Ohlyac45f602009-02-12 05:03:37 +00002161 rc = ops->ndo_start_xmit(skb, dev);
Koki Sanagicf66ba52010-08-23 18:45:02 +09002162 trace_net_dev_xmit(skb, rc);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002163 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07002164 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00002165 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002166 }
2167
Herbert Xu576a30e2006-06-27 13:22:38 -07002168gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002169 do {
2170 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002171
2172 skb->next = nskb->next;
2173 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00002174
2175 /*
2176 * If device doesnt need nskb->dst, release it right now while
2177 * its hot in this cpu cache
2178 */
2179 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2180 skb_dst_drop(nskb);
2181
Stephen Hemminger00829822008-11-20 20:14:53 -08002182 rc = ops->ndo_start_xmit(nskb, dev);
Koki Sanagicf66ba52010-08-23 18:45:02 +09002183 trace_net_dev_xmit(nskb, rc);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002184 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002185 if (rc & ~NETDEV_TX_MASK)
2186 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07002187 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002188 skb->next = nskb;
2189 return rc;
2190 }
Eric Dumazet08baf562009-05-25 22:58:01 -07002191 txq_trans_update(txq);
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002192 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07002193 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002194 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002195
Patrick McHardy572a9d72009-11-10 06:14:14 +00002196out_kfree_gso_skb:
2197 if (likely(skb->next == NULL))
2198 skb->destructor = DEV_GSO_CB(skb)->destructor;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002199out_kfree_skb:
2200 kfree_skb(skb);
Jesse Gross7b9c6092010-10-20 13:56:04 +00002201out:
Patrick McHardy572a9d72009-11-10 06:14:14 +00002202 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002203}
2204
Tom Herbert0a9627f2010-03-16 08:03:29 +00002205static u32 hashrnd __read_mostly;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002206
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002207/*
2208 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2209 * to be used as a distribution range.
2210 */
2211u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2212 unsigned int num_tx_queues)
David S. Miller8f0f2222008-07-15 03:47:03 -07002213{
David S. Miller70192982009-01-27 16:34:47 -08002214 u32 hash;
John Fastabend4f57c082011-01-17 08:06:04 +00002215 u16 qoffset = 0;
2216 u16 qcount = num_tx_queues;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002217
David S. Miller513de112009-05-03 14:43:10 -07002218 if (skb_rx_queue_recorded(skb)) {
2219 hash = skb_get_rx_queue(skb);
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002220 while (unlikely(hash >= num_tx_queues))
2221 hash -= num_tx_queues;
David S. Miller513de112009-05-03 14:43:10 -07002222 return hash;
2223 }
Eric Dumazetec581f62009-05-01 09:05:06 -07002224
John Fastabend4f57c082011-01-17 08:06:04 +00002225 if (dev->num_tc) {
2226 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2227 qoffset = dev->tc_to_txq[tc].offset;
2228 qcount = dev->tc_to_txq[tc].count;
2229 }
2230
Eric Dumazetec581f62009-05-01 09:05:06 -07002231 if (skb->sk && skb->sk->sk_hash)
David S. Miller70192982009-01-27 16:34:47 -08002232 hash = skb->sk->sk_hash;
Eric Dumazetec581f62009-05-01 09:05:06 -07002233 else
Eric Dumazet87fd3082010-07-13 05:24:20 +00002234 hash = (__force u16) skb->protocol ^ skb->rxhash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002235 hash = jhash_1word(hash, hashrnd);
David S. Millerd5a9e242009-01-27 16:22:11 -08002236
John Fastabend4f57c082011-01-17 08:06:04 +00002237 return (u16) (((u64) hash * qcount) >> 32) + qoffset;
David S. Miller8f0f2222008-07-15 03:47:03 -07002238}
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002239EXPORT_SYMBOL(__skb_tx_hash);
David S. Miller8f0f2222008-07-15 03:47:03 -07002240
Eric Dumazeted046422009-11-13 21:54:04 +00002241static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2242{
2243 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2244 if (net_ratelimit()) {
Eric Dumazet7a161ea2010-04-08 21:26:13 +00002245 pr_warning("%s selects TX queue %d, but "
2246 "real number of TX queues is %d\n",
2247 dev->name, queue_index, dev->real_num_tx_queues);
Eric Dumazeted046422009-11-13 21:54:04 +00002248 }
2249 return 0;
2250 }
2251 return queue_index;
2252}
2253
Tom Herbert1d24eb42010-11-21 13:17:27 +00002254static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2255{
Tom Herbertbf264142010-11-26 08:36:09 +00002256#ifdef CONFIG_XPS
Tom Herbert1d24eb42010-11-21 13:17:27 +00002257 struct xps_dev_maps *dev_maps;
2258 struct xps_map *map;
2259 int queue_index = -1;
2260
2261 rcu_read_lock();
2262 dev_maps = rcu_dereference(dev->xps_maps);
2263 if (dev_maps) {
2264 map = rcu_dereference(
2265 dev_maps->cpu_map[raw_smp_processor_id()]);
2266 if (map) {
2267 if (map->len == 1)
2268 queue_index = map->queues[0];
2269 else {
2270 u32 hash;
2271 if (skb->sk && skb->sk->sk_hash)
2272 hash = skb->sk->sk_hash;
2273 else
2274 hash = (__force u16) skb->protocol ^
2275 skb->rxhash;
2276 hash = jhash_1word(hash, hashrnd);
2277 queue_index = map->queues[
2278 ((u64)hash * map->len) >> 32];
2279 }
2280 if (unlikely(queue_index >= dev->real_num_tx_queues))
2281 queue_index = -1;
2282 }
2283 }
2284 rcu_read_unlock();
2285
2286 return queue_index;
2287#else
2288 return -1;
2289#endif
2290}
2291
David S. Millere8a04642008-07-17 00:34:19 -07002292static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2293 struct sk_buff *skb)
2294{
Tom Herbertb0f77d02010-07-14 20:50:29 -07002295 int queue_index;
Helmut Schaadeabc772010-09-03 02:39:56 +00002296 const struct net_device_ops *ops = dev->netdev_ops;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002297
Tom Herbert3853b582010-11-21 13:17:29 +00002298 if (dev->real_num_tx_queues == 1)
2299 queue_index = 0;
2300 else if (ops->ndo_select_queue) {
Helmut Schaadeabc772010-09-03 02:39:56 +00002301 queue_index = ops->ndo_select_queue(dev, skb);
2302 queue_index = dev_cap_txqueue(dev, queue_index);
2303 } else {
2304 struct sock *sk = skb->sk;
2305 queue_index = sk_tx_queue_get(sk);
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002306
Tom Herbert3853b582010-11-21 13:17:29 +00002307 if (queue_index < 0 || skb->ooo_okay ||
2308 queue_index >= dev->real_num_tx_queues) {
2309 int old_index = queue_index;
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002310
Tom Herbert1d24eb42010-11-21 13:17:27 +00002311 queue_index = get_xps_queue(dev, skb);
2312 if (queue_index < 0)
2313 queue_index = skb_tx_hash(dev, skb);
Tom Herbert3853b582010-11-21 13:17:29 +00002314
2315 if (queue_index != old_index && sk) {
2316 struct dst_entry *dst =
2317 rcu_dereference_check(sk->sk_dst_cache, 1);
Eric Dumazet8728c542010-04-11 21:18:17 +00002318
2319 if (dst && skb_dst(skb) == dst)
2320 sk_tx_queue_set(sk, queue_index);
2321 }
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002322 }
2323 }
David S. Millereae792b2008-07-15 03:03:33 -07002324
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002325 skb_set_queue_mapping(skb, queue_index);
2326 return netdev_get_tx_queue(dev, queue_index);
David S. Millere8a04642008-07-17 00:34:19 -07002327}
2328
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002329static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2330 struct net_device *dev,
2331 struct netdev_queue *txq)
2332{
2333 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002334 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002335 int rc;
2336
Eric Dumazeta2da5702011-01-20 03:48:19 +00002337 qdisc_skb_cb(skb)->pkt_len = skb->len;
2338 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002339 /*
2340 * Heuristic to force contended enqueues to serialize on a
2341 * separate lock before trying to get qdisc main lock.
2342 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2343 * and dequeue packets faster.
2344 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002345 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002346 if (unlikely(contended))
2347 spin_lock(&q->busylock);
2348
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002349 spin_lock(root_lock);
2350 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2351 kfree_skb(skb);
2352 rc = NET_XMIT_DROP;
2353 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002354 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002355 /*
2356 * This is a work-conserving queue; there are no old skbs
2357 * waiting to be sent out; and the qdisc is not running -
2358 * xmit the skb directly.
2359 */
Eric Dumazet7fee2262010-05-11 23:19:48 +00002360 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2361 skb_dst_force(skb);
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002362
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002363 qdisc_bstats_update(q, skb);
2364
Eric Dumazet79640a42010-06-02 05:09:29 -07002365 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2366 if (unlikely(contended)) {
2367 spin_unlock(&q->busylock);
2368 contended = false;
2369 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002370 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002371 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002372 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002373
2374 rc = NET_XMIT_SUCCESS;
2375 } else {
Eric Dumazet7fee2262010-05-11 23:19:48 +00002376 skb_dst_force(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002377 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002378 if (qdisc_run_begin(q)) {
2379 if (unlikely(contended)) {
2380 spin_unlock(&q->busylock);
2381 contended = false;
2382 }
2383 __qdisc_run(q);
2384 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002385 }
2386 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002387 if (unlikely(contended))
2388 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002389 return rc;
2390}
2391
Eric Dumazet745e20f2010-09-29 13:23:09 -07002392static DEFINE_PER_CPU(int, xmit_recursion);
David S. Miller11a766c2010-10-25 12:51:55 -07002393#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002394
Dave Jonesd29f7492008-07-22 14:09:06 -07002395/**
2396 * dev_queue_xmit - transmit a buffer
2397 * @skb: buffer to transmit
2398 *
2399 * Queue a buffer for transmission to a network device. The caller must
2400 * have set the device and priority and built the buffer before calling
2401 * this function. The function can be called from an interrupt.
2402 *
2403 * A negative errno code is returned on a failure. A success does not
2404 * guarantee the frame will be transmitted as it may be dropped due
2405 * to congestion or traffic shaping.
2406 *
2407 * -----------------------------------------------------------------------------------
2408 * I notice this method can also return errors from the queue disciplines,
2409 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2410 * be positive.
2411 *
2412 * Regardless of the return value, the skb is consumed, so it is currently
2413 * difficult to retry a send to this method. (You can bump the ref count
2414 * before sending to hold a reference for retry if you are careful.)
2415 *
2416 * When calling this method, interrupts MUST be enabled. This is because
2417 * the BH enable code must have IRQs enabled so that it will not deadlock.
2418 * --BLG
2419 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002420int dev_queue_xmit(struct sk_buff *skb)
2421{
2422 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002423 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002424 struct Qdisc *q;
2425 int rc = -ENOMEM;
2426
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002427 /* Disable soft irqs for various locks below. Also
2428 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002429 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002430 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002431
David S. Millereae792b2008-07-15 03:03:33 -07002432 txq = dev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002433 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002434
Linus Torvalds1da177e2005-04-16 15:20:36 -07002435#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002436 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002437#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002438 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002439 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002440 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002441 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002442 }
2443
2444 /* The device has no queue. Common case for software devices:
2445 loopback, all the sorts of tunnels...
2446
Herbert Xu932ff272006-06-09 12:20:56 -07002447 Really, it is unlikely that netif_tx_lock protection is necessary
2448 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449 counters.)
2450 However, it is possible, that they rely on protection
2451 made by us here.
2452
2453 Check this and shot the lock. It is not prone from deadlocks.
2454 Either shot noqueue qdisc, it is even simpler 8)
2455 */
2456 if (dev->flags & IFF_UP) {
2457 int cpu = smp_processor_id(); /* ok because BHs are off */
2458
David S. Millerc773e842008-07-08 23:13:53 -07002459 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002460
Eric Dumazet745e20f2010-09-29 13:23:09 -07002461 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2462 goto recursion_alert;
2463
David S. Millerc773e842008-07-08 23:13:53 -07002464 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002465
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002466 if (!netif_tx_queue_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002467 __this_cpu_inc(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002468 rc = dev_hard_start_xmit(skb, dev, txq);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002469 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002470 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002471 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002472 goto out;
2473 }
2474 }
David S. Millerc773e842008-07-08 23:13:53 -07002475 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002476 if (net_ratelimit())
2477 printk(KERN_CRIT "Virtual device %s asks to "
2478 "queue packet!\n", dev->name);
2479 } else {
2480 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07002481 * unfortunately
2482 */
2483recursion_alert:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002484 if (net_ratelimit())
2485 printk(KERN_CRIT "Dead loop on virtual device "
2486 "%s, fix it urgently!\n", dev->name);
2487 }
2488 }
2489
2490 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002491 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002492
Linus Torvalds1da177e2005-04-16 15:20:36 -07002493 kfree_skb(skb);
2494 return rc;
2495out:
Herbert Xud4828d82006-06-22 02:28:18 -07002496 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002497 return rc;
2498}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002499EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002500
2501
2502/*=======================================================================
2503 Receiver routines
2504 =======================================================================*/
2505
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002506int netdev_max_backlog __read_mostly = 1000;
Eric Dumazet3b098e22010-05-15 23:57:10 -07002507int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002508int netdev_budget __read_mostly = 300;
2509int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002510
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002511/* Called with irq disabled */
2512static inline void ____napi_schedule(struct softnet_data *sd,
2513 struct napi_struct *napi)
2514{
2515 list_add_tail(&napi->poll_list, &sd->poll_list);
2516 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2517}
2518
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002519/*
2520 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
2521 * and src/dst port numbers. Returns a non-zero hash number on success
2522 * and 0 on failure.
2523 */
2524__u32 __skb_get_rxhash(struct sk_buff *skb)
2525{
Changli Gao12fcdef2010-08-17 19:04:32 +00002526 int nhoff, hash = 0, poff;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002527 struct ipv6hdr *ip6;
2528 struct iphdr *ip;
2529 u8 ip_proto;
2530 u32 addr1, addr2, ihl;
2531 union {
2532 u32 v32;
2533 u16 v16[2];
2534 } ports;
2535
2536 nhoff = skb_network_offset(skb);
2537
2538 switch (skb->protocol) {
2539 case __constant_htons(ETH_P_IP):
2540 if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))
2541 goto done;
2542
Changli Gao10034892010-08-21 06:13:28 +00002543 ip = (struct iphdr *) (skb->data + nhoff);
Changli Gaodbe57752010-08-17 19:01:38 +00002544 if (ip->frag_off & htons(IP_MF | IP_OFFSET))
2545 ip_proto = 0;
2546 else
2547 ip_proto = ip->protocol;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002548 addr1 = (__force u32) ip->saddr;
2549 addr2 = (__force u32) ip->daddr;
2550 ihl = ip->ihl;
2551 break;
2552 case __constant_htons(ETH_P_IPV6):
2553 if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))
2554 goto done;
2555
Changli Gao10034892010-08-21 06:13:28 +00002556 ip6 = (struct ipv6hdr *) (skb->data + nhoff);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002557 ip_proto = ip6->nexthdr;
2558 addr1 = (__force u32) ip6->saddr.s6_addr32[3];
2559 addr2 = (__force u32) ip6->daddr.s6_addr32[3];
2560 ihl = (40 >> 2);
2561 break;
2562 default:
2563 goto done;
2564 }
2565
Changli Gao12fcdef2010-08-17 19:04:32 +00002566 ports.v32 = 0;
2567 poff = proto_ports_offset(ip_proto);
2568 if (poff >= 0) {
2569 nhoff += ihl * 4 + poff;
2570 if (pskb_may_pull(skb, nhoff + 4)) {
2571 ports.v32 = * (__force u32 *) (skb->data + nhoff);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002572 if (ports.v16[1] < ports.v16[0])
2573 swap(ports.v16[0], ports.v16[1]);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002574 }
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002575 }
2576
2577 /* get a consistent hash (same value on both flow directions) */
2578 if (addr2 < addr1)
2579 swap(addr1, addr2);
2580
2581 hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);
2582 if (!hash)
2583 hash = 1;
2584
2585done:
2586 return hash;
2587}
2588EXPORT_SYMBOL(__skb_get_rxhash);
2589
Eric Dumazetdf334542010-03-24 19:13:54 +00002590#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07002591
2592/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002593struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07002594EXPORT_SYMBOL(rps_sock_flow_table);
2595
Ben Hutchingsc4454772011-01-19 11:03:53 +00002596static struct rps_dev_flow *
2597set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2598 struct rps_dev_flow *rflow, u16 next_cpu)
2599{
2600 u16 tcpu;
2601
2602 tcpu = rflow->cpu = next_cpu;
2603 if (tcpu != RPS_NO_CPU) {
2604#ifdef CONFIG_RFS_ACCEL
2605 struct netdev_rx_queue *rxqueue;
2606 struct rps_dev_flow_table *flow_table;
2607 struct rps_dev_flow *old_rflow;
2608 u32 flow_id;
2609 u16 rxq_index;
2610 int rc;
2611
2612 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00002613 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2614 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00002615 goto out;
2616 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2617 if (rxq_index == skb_get_rx_queue(skb))
2618 goto out;
2619
2620 rxqueue = dev->_rx + rxq_index;
2621 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2622 if (!flow_table)
2623 goto out;
2624 flow_id = skb->rxhash & flow_table->mask;
2625 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2626 rxq_index, flow_id);
2627 if (rc < 0)
2628 goto out;
2629 old_rflow = rflow;
2630 rflow = &flow_table->flows[flow_id];
2631 rflow->cpu = next_cpu;
2632 rflow->filter = rc;
2633 if (old_rflow->filter == rflow->filter)
2634 old_rflow->filter = RPS_NO_FILTER;
2635 out:
2636#endif
2637 rflow->last_qtail =
2638 per_cpu(softnet_data, tcpu).input_queue_head;
2639 }
2640
2641 return rflow;
2642}
2643
Tom Herbert0a9627f2010-03-16 08:03:29 +00002644/*
2645 * get_rps_cpu is called from netif_receive_skb and returns the target
2646 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002647 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00002648 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002649static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2650 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002651{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002652 struct netdev_rx_queue *rxqueue;
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002653 struct rps_map *map;
Tom Herbertfec5e652010-04-16 16:01:27 -07002654 struct rps_dev_flow_table *flow_table;
2655 struct rps_sock_flow_table *sock_flow_table;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002656 int cpu = -1;
Tom Herbertfec5e652010-04-16 16:01:27 -07002657 u16 tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002658
Tom Herbert0a9627f2010-03-16 08:03:29 +00002659 if (skb_rx_queue_recorded(skb)) {
2660 u16 index = skb_get_rx_queue(skb);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002661 if (unlikely(index >= dev->real_num_rx_queues)) {
2662 WARN_ONCE(dev->real_num_rx_queues > 1,
2663 "%s received packet on queue %u, but number "
2664 "of RX queues is %u\n",
2665 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002666 goto done;
2667 }
2668 rxqueue = dev->_rx + index;
2669 } else
2670 rxqueue = dev->_rx;
2671
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002672 map = rcu_dereference(rxqueue->rps_map);
2673 if (map) {
Tom Herbert85875232011-01-31 16:23:42 -08002674 if (map->len == 1 &&
2675 !rcu_dereference_raw(rxqueue->rps_flow_table)) {
Changli Gao6febfca2010-09-03 23:12:37 +00002676 tcpu = map->cpus[0];
2677 if (cpu_online(tcpu))
2678 cpu = tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002679 goto done;
Eric Dumazetb249dcb2010-04-19 21:56:38 +00002680 }
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002681 } else if (!rcu_dereference_raw(rxqueue->rps_flow_table)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002682 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002683 }
2684
Changli Gao2d47b452010-08-17 19:00:56 +00002685 skb_reset_network_header(skb);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002686 if (!skb_get_rxhash(skb))
Tom Herbert0a9627f2010-03-16 08:03:29 +00002687 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002688
Tom Herbertfec5e652010-04-16 16:01:27 -07002689 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2690 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2691 if (flow_table && sock_flow_table) {
2692 u16 next_cpu;
2693 struct rps_dev_flow *rflow;
2694
2695 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2696 tcpu = rflow->cpu;
2697
2698 next_cpu = sock_flow_table->ents[skb->rxhash &
2699 sock_flow_table->mask];
2700
2701 /*
2702 * If the desired CPU (where last recvmsg was done) is
2703 * different from current CPU (one in the rx-queue flow
2704 * table entry), switch if one of the following holds:
2705 * - Current CPU is unset (equal to RPS_NO_CPU).
2706 * - Current CPU is offline.
2707 * - The current CPU's queue tail has advanced beyond the
2708 * last packet that was enqueued using this table entry.
2709 * This guarantees that all previous packets for the flow
2710 * have been dequeued, thus preserving in order delivery.
2711 */
2712 if (unlikely(tcpu != next_cpu) &&
2713 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2714 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Ben Hutchingsc4454772011-01-19 11:03:53 +00002715 rflow->last_qtail)) >= 0))
2716 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
2717
Tom Herbertfec5e652010-04-16 16:01:27 -07002718 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2719 *rflowp = rflow;
2720 cpu = tcpu;
2721 goto done;
2722 }
2723 }
2724
Tom Herbert0a9627f2010-03-16 08:03:29 +00002725 if (map) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002726 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
Tom Herbert0a9627f2010-03-16 08:03:29 +00002727
2728 if (cpu_online(tcpu)) {
2729 cpu = tcpu;
2730 goto done;
2731 }
2732 }
2733
2734done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00002735 return cpu;
2736}
2737
Ben Hutchingsc4454772011-01-19 11:03:53 +00002738#ifdef CONFIG_RFS_ACCEL
2739
2740/**
2741 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
2742 * @dev: Device on which the filter was set
2743 * @rxq_index: RX queue index
2744 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
2745 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
2746 *
2747 * Drivers that implement ndo_rx_flow_steer() should periodically call
2748 * this function for each installed filter and remove the filters for
2749 * which it returns %true.
2750 */
2751bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
2752 u32 flow_id, u16 filter_id)
2753{
2754 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
2755 struct rps_dev_flow_table *flow_table;
2756 struct rps_dev_flow *rflow;
2757 bool expire = true;
2758 int cpu;
2759
2760 rcu_read_lock();
2761 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2762 if (flow_table && flow_id <= flow_table->mask) {
2763 rflow = &flow_table->flows[flow_id];
2764 cpu = ACCESS_ONCE(rflow->cpu);
2765 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
2766 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
2767 rflow->last_qtail) <
2768 (int)(10 * flow_table->mask)))
2769 expire = false;
2770 }
2771 rcu_read_unlock();
2772 return expire;
2773}
2774EXPORT_SYMBOL(rps_may_expire_flow);
2775
2776#endif /* CONFIG_RFS_ACCEL */
2777
Tom Herbert0a9627f2010-03-16 08:03:29 +00002778/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002779static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002780{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002781 struct softnet_data *sd = data;
2782
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002783 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00002784 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002785}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002786
Tom Herbertfec5e652010-04-16 16:01:27 -07002787#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00002788
2789/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002790 * Check if this softnet_data structure is another cpu one
2791 * If yes, queue it to our IPI list and return 1
2792 * If no, return 0
2793 */
2794static int rps_ipi_queued(struct softnet_data *sd)
2795{
2796#ifdef CONFIG_RPS
2797 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2798
2799 if (sd != mysd) {
2800 sd->rps_ipi_next = mysd->rps_ipi_list;
2801 mysd->rps_ipi_list = sd;
2802
2803 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2804 return 1;
2805 }
2806#endif /* CONFIG_RPS */
2807 return 0;
2808}
2809
2810/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00002811 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2812 * queue (may be a remote CPU queue).
2813 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002814static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2815 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002816{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002817 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002818 unsigned long flags;
2819
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002820 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002821
2822 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002823
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002824 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07002825 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2826 if (skb_queue_len(&sd->input_pkt_queue)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002827enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002828 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00002829 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002830 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00002831 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002832 return NET_RX_SUCCESS;
2833 }
2834
Eric Dumazetebda37c22010-05-06 23:51:21 +00002835 /* Schedule NAPI for backlog device
2836 * We can use non atomic operation since we own the queue lock
2837 */
2838 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002839 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002840 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002841 }
2842 goto enqueue;
2843 }
2844
Changli Gaodee42872010-05-02 05:42:16 +00002845 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002846 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002847
Tom Herbert0a9627f2010-03-16 08:03:29 +00002848 local_irq_restore(flags);
2849
Eric Dumazetcaf586e2010-09-30 21:06:55 +00002850 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002851 kfree_skb(skb);
2852 return NET_RX_DROP;
2853}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002854
Linus Torvalds1da177e2005-04-16 15:20:36 -07002855/**
2856 * netif_rx - post buffer to the network code
2857 * @skb: buffer to post
2858 *
2859 * This function receives a packet from a device driver and queues it for
2860 * the upper (protocol) levels to process. It always succeeds. The buffer
2861 * may be dropped during processing for congestion control or by the
2862 * protocol layers.
2863 *
2864 * return values:
2865 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002866 * NET_RX_DROP (packet was dropped)
2867 *
2868 */
2869
2870int netif_rx(struct sk_buff *skb)
2871{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002872 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002873
2874 /* if netpoll wants it, pretend we never saw it */
2875 if (netpoll_rx(skb))
2876 return NET_RX_DROP;
2877
Eric Dumazet3b098e22010-05-15 23:57:10 -07002878 if (netdev_tstamp_prequeue)
2879 net_timestamp_check(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002880
Koki Sanagicf66ba52010-08-23 18:45:02 +09002881 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00002882#ifdef CONFIG_RPS
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002883 {
Tom Herbertfec5e652010-04-16 16:01:27 -07002884 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002885 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002886
Changli Gaocece1942010-08-07 20:35:43 -07002887 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002888 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07002889
2890 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002891 if (cpu < 0)
2892 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07002893
2894 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2895
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002896 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07002897 preempt_enable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002898 }
2899#else
Tom Herbertfec5e652010-04-16 16:01:27 -07002900 {
2901 unsigned int qtail;
2902 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
2903 put_cpu();
2904 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002905#endif
2906 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002907}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002908EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002909
2910int netif_rx_ni(struct sk_buff *skb)
2911{
2912 int err;
2913
2914 preempt_disable();
2915 err = netif_rx(skb);
2916 if (local_softirq_pending())
2917 do_softirq();
2918 preempt_enable();
2919
2920 return err;
2921}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002922EXPORT_SYMBOL(netif_rx_ni);
2923
Linus Torvalds1da177e2005-04-16 15:20:36 -07002924static void net_tx_action(struct softirq_action *h)
2925{
2926 struct softnet_data *sd = &__get_cpu_var(softnet_data);
2927
2928 if (sd->completion_queue) {
2929 struct sk_buff *clist;
2930
2931 local_irq_disable();
2932 clist = sd->completion_queue;
2933 sd->completion_queue = NULL;
2934 local_irq_enable();
2935
2936 while (clist) {
2937 struct sk_buff *skb = clist;
2938 clist = clist->next;
2939
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002940 WARN_ON(atomic_read(&skb->users));
Koki Sanagi07dc22e2010-08-23 18:46:12 +09002941 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002942 __kfree_skb(skb);
2943 }
2944 }
2945
2946 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07002947 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002948
2949 local_irq_disable();
2950 head = sd->output_queue;
2951 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00002952 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002953 local_irq_enable();
2954
2955 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07002956 struct Qdisc *q = head;
2957 spinlock_t *root_lock;
2958
Linus Torvalds1da177e2005-04-16 15:20:36 -07002959 head = head->next_sched;
2960
David S. Miller5fb66222008-08-02 20:02:43 -07002961 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07002962 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002963 smp_mb__before_clear_bit();
2964 clear_bit(__QDISC_STATE_SCHED,
2965 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07002966 qdisc_run(q);
2967 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002968 } else {
David S. Miller195648b2008-08-19 04:00:36 -07002969 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07002970 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07002971 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07002972 } else {
2973 smp_mb__before_clear_bit();
2974 clear_bit(__QDISC_STATE_SCHED,
2975 &q->state);
2976 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002977 }
2978 }
2979 }
2980}
2981
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00002982#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
2983 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00002984/* This hook is defined here for ATM LANE */
2985int (*br_fdb_test_addr_hook)(struct net_device *dev,
2986 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07002987EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00002988#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002989
Linus Torvalds1da177e2005-04-16 15:20:36 -07002990#ifdef CONFIG_NET_CLS_ACT
2991/* TODO: Maybe we should just force sch_ingress to be compiled in
2992 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
2993 * a compare and 2 stores extra right now if we dont have it on
2994 * but have CONFIG_NET_CLS_ACT
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002995 * NOTE: This doesnt stop any functionality; if you dont have
Linus Torvalds1da177e2005-04-16 15:20:36 -07002996 * the ingress scheduler, you just cant add policies on ingress.
2997 *
2998 */
Eric Dumazet24824a02010-10-02 06:11:55 +00002999static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003000{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003001 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003002 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07003003 int result = TC_ACT_OK;
3004 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003005
Stephen Hemmingerde384832010-08-01 00:33:23 -07003006 if (unlikely(MAX_RED_LOOP < ttl++)) {
3007 if (net_ratelimit())
3008 pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",
3009 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07003010 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003011 }
3012
Herbert Xuf697c3e2007-10-14 00:38:47 -07003013 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3014 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3015
David S. Miller83874002008-07-17 00:53:03 -07003016 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07003017 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07003018 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07003019 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3020 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07003021 spin_unlock(qdisc_lock(q));
3022 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07003023
Linus Torvalds1da177e2005-04-16 15:20:36 -07003024 return result;
3025}
Herbert Xuf697c3e2007-10-14 00:38:47 -07003026
3027static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3028 struct packet_type **pt_prev,
3029 int *ret, struct net_device *orig_dev)
3030{
Eric Dumazet24824a02010-10-02 06:11:55 +00003031 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3032
3033 if (!rxq || rxq->qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003034 goto out;
3035
3036 if (*pt_prev) {
3037 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3038 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003039 }
3040
Eric Dumazet24824a02010-10-02 06:11:55 +00003041 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07003042 case TC_ACT_SHOT:
3043 case TC_ACT_STOLEN:
3044 kfree_skb(skb);
3045 return NULL;
3046 }
3047
3048out:
3049 skb->tc_verd = 0;
3050 return skb;
3051}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003052#endif
3053
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003054/**
3055 * netdev_rx_handler_register - register receive handler
3056 * @dev: device to register a handler for
3057 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003058 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003059 *
3060 * Register a receive hander for a device. This handler will then be
3061 * called from __netif_receive_skb. A negative errno code is returned
3062 * on a failure.
3063 *
3064 * The caller must hold the rtnl_mutex.
3065 */
3066int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003067 rx_handler_func_t *rx_handler,
3068 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003069{
3070 ASSERT_RTNL();
3071
3072 if (dev->rx_handler)
3073 return -EBUSY;
3074
Jiri Pirko93e2c322010-06-10 03:34:59 +00003075 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003076 rcu_assign_pointer(dev->rx_handler, rx_handler);
3077
3078 return 0;
3079}
3080EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3081
3082/**
3083 * netdev_rx_handler_unregister - unregister receive handler
3084 * @dev: device to unregister a handler from
3085 *
3086 * Unregister a receive hander from a device.
3087 *
3088 * The caller must hold the rtnl_mutex.
3089 */
3090void netdev_rx_handler_unregister(struct net_device *dev)
3091{
3092
3093 ASSERT_RTNL();
3094 rcu_assign_pointer(dev->rx_handler, NULL);
Jiri Pirko93e2c322010-06-10 03:34:59 +00003095 rcu_assign_pointer(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003096}
3097EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3098
David S. Miller63d8ea72011-02-28 10:48:59 -08003099static void vlan_on_bond_hook(struct sk_buff *skb)
Eric Dumazetacbbc072010-04-11 06:56:11 +00003100{
David S. Miller63d8ea72011-02-28 10:48:59 -08003101 /*
3102 * Make sure ARP frames received on VLAN interfaces stacked on
3103 * bonding interfaces still make their way to any base bonding
3104 * device that may have registered for a specific ptype.
3105 */
3106 if (skb->dev->priv_flags & IFF_802_1Q_VLAN &&
3107 vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING &&
3108 skb->protocol == htons(ETH_P_ARP)) {
3109 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
Eric Dumazetacbbc072010-04-11 06:56:11 +00003110
David S. Miller63d8ea72011-02-28 10:48:59 -08003111 if (!skb2)
3112 return;
3113 skb2->dev = vlan_dev_real_dev(skb->dev);
3114 netif_rx(skb2);
Eric Dumazetacbbc072010-04-11 06:56:11 +00003115 }
3116}
3117
Eric Dumazet10f744d2010-03-28 23:07:20 -07003118static int __netif_receive_skb(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003119{
3120 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003121 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003122 struct net_device *orig_dev;
David S. Miller63d8ea72011-02-28 10:48:59 -08003123 struct net_device *null_or_dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003124 int ret = NET_RX_DROP;
Al Viro252e33462006-11-14 20:48:11 -08003125 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003126
Eric Dumazet3b098e22010-05-15 23:57:10 -07003127 if (!netdev_tstamp_prequeue)
3128 net_timestamp_check(skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003129
Koki Sanagicf66ba52010-08-23 18:45:02 +09003130 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003131
Linus Torvalds1da177e2005-04-16 15:20:36 -07003132 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003133 if (netpoll_receive_skb(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003134 return NET_RX_DROP;
3135
Eric Dumazet8964be42009-11-20 15:35:04 -08003136 if (!skb->skb_iif)
3137 skb->skb_iif = skb->dev->ifindex;
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003138 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003139
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003140 skb_reset_network_header(skb);
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -03003141 skb_reset_transport_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07003142 skb->mac_len = skb->network_header - skb->mac_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003143
3144 pt_prev = NULL;
3145
3146 rcu_read_lock();
3147
David S. Miller63d8ea72011-02-28 10:48:59 -08003148another_round:
3149
3150 __this_cpu_inc(softnet_data.processed);
3151
Linus Torvalds1da177e2005-04-16 15:20:36 -07003152#ifdef CONFIG_NET_CLS_ACT
3153 if (skb->tc_verd & TC_NCLS) {
3154 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3155 goto ncls;
3156 }
3157#endif
3158
3159 list_for_each_entry_rcu(ptype, &ptype_all, list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003160 if (!ptype->dev || ptype->dev == skb->dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003161 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003162 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003163 pt_prev = ptype;
3164 }
3165 }
3166
3167#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07003168 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3169 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003170 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003171ncls:
3172#endif
3173
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003174 rx_handler = rcu_dereference(skb->dev->rx_handler);
3175 if (rx_handler) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003176 struct net_device *prev_dev;
3177
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003178 if (pt_prev) {
3179 ret = deliver_skb(skb, pt_prev, orig_dev);
3180 pt_prev = NULL;
3181 }
David S. Miller63d8ea72011-02-28 10:48:59 -08003182 prev_dev = skb->dev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003183 skb = rx_handler(skb);
3184 if (!skb)
3185 goto out;
David S. Miller63d8ea72011-02-28 10:48:59 -08003186 if (skb->dev != prev_dev)
3187 goto another_round;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003188 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003189
Jesse Gross3701e512010-10-20 13:56:06 +00003190 if (vlan_tx_tag_present(skb)) {
3191 if (pt_prev) {
3192 ret = deliver_skb(skb, pt_prev, orig_dev);
3193 pt_prev = NULL;
3194 }
3195 if (vlan_hwaccel_do_receive(&skb)) {
3196 ret = __netif_receive_skb(skb);
3197 goto out;
3198 } else if (unlikely(!skb))
3199 goto out;
3200 }
3201
David S. Miller63d8ea72011-02-28 10:48:59 -08003202 vlan_on_bond_hook(skb);
3203
3204 /* deliver only exact match when indicated */
3205 null_or_dev = skb->deliver_no_wcard ? skb->dev : NULL;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00003206
Linus Torvalds1da177e2005-04-16 15:20:36 -07003207 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003208 list_for_each_entry_rcu(ptype,
3209 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003210 if (ptype->type == type &&
Jiri Pirkoe3f48d32011-02-28 20:26:31 +00003211 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3212 ptype->dev == orig_dev)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003213 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003214 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003215 pt_prev = ptype;
3216 }
3217 }
3218
3219 if (pt_prev) {
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003220 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003221 } else {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003222 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003223 kfree_skb(skb);
3224 /* Jamal, now you will not able to escape explaining
3225 * me how you were going to use this. :-)
3226 */
3227 ret = NET_RX_DROP;
3228 }
3229
3230out:
3231 rcu_read_unlock();
3232 return ret;
3233}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003234
3235/**
3236 * netif_receive_skb - process receive buffer from network
3237 * @skb: buffer to process
3238 *
3239 * netif_receive_skb() is the main receive data processing function.
3240 * It always succeeds. The buffer may be dropped during processing
3241 * for congestion control or by the protocol layers.
3242 *
3243 * This function may only be called from softirq context and interrupts
3244 * should be enabled.
3245 *
3246 * Return values (usually ignored):
3247 * NET_RX_SUCCESS: no congestion
3248 * NET_RX_DROP: packet was dropped
3249 */
3250int netif_receive_skb(struct sk_buff *skb)
3251{
Eric Dumazet3b098e22010-05-15 23:57:10 -07003252 if (netdev_tstamp_prequeue)
3253 net_timestamp_check(skb);
3254
Richard Cochranc1f19b52010-07-17 08:49:36 +00003255 if (skb_defer_rx_timestamp(skb))
3256 return NET_RX_SUCCESS;
3257
Eric Dumazetdf334542010-03-24 19:13:54 +00003258#ifdef CONFIG_RPS
Eric Dumazet3b098e22010-05-15 23:57:10 -07003259 {
3260 struct rps_dev_flow voidflow, *rflow = &voidflow;
3261 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003262
Eric Dumazet3b098e22010-05-15 23:57:10 -07003263 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003264
Eric Dumazet3b098e22010-05-15 23:57:10 -07003265 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003266
Eric Dumazet3b098e22010-05-15 23:57:10 -07003267 if (cpu >= 0) {
3268 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3269 rcu_read_unlock();
3270 } else {
3271 rcu_read_unlock();
3272 ret = __netif_receive_skb(skb);
3273 }
3274
3275 return ret;
Tom Herbertfec5e652010-04-16 16:01:27 -07003276 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003277#else
3278 return __netif_receive_skb(skb);
3279#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00003280}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003281EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003282
Eric Dumazet88751272010-04-19 05:07:33 +00003283/* Network device is going away, flush any packets still pending
3284 * Called with irqs disabled.
3285 */
Changli Gao152102c2010-03-30 20:16:22 +00003286static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003287{
Changli Gao152102c2010-03-30 20:16:22 +00003288 struct net_device *dev = arg;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003289 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003290 struct sk_buff *skb, *tmp;
3291
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003292 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003293 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003294 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003295 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003296 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003297 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003298 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003299 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003300 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003301
3302 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3303 if (skb->dev == dev) {
3304 __skb_unlink(skb, &sd->process_queue);
3305 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003306 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003307 }
3308 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003309}
3310
Herbert Xud565b0a2008-12-15 23:38:52 -08003311static int napi_gro_complete(struct sk_buff *skb)
3312{
3313 struct packet_type *ptype;
3314 __be16 type = skb->protocol;
3315 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3316 int err = -ENOENT;
3317
Herbert Xufc59f9a2009-04-14 15:11:06 -07003318 if (NAPI_GRO_CB(skb)->count == 1) {
3319 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003320 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003321 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003322
3323 rcu_read_lock();
3324 list_for_each_entry_rcu(ptype, head, list) {
3325 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3326 continue;
3327
3328 err = ptype->gro_complete(skb);
3329 break;
3330 }
3331 rcu_read_unlock();
3332
3333 if (err) {
3334 WARN_ON(&ptype->list == head);
3335 kfree_skb(skb);
3336 return NET_RX_SUCCESS;
3337 }
3338
3339out:
Herbert Xud565b0a2008-12-15 23:38:52 -08003340 return netif_receive_skb(skb);
3341}
3342
Eric Dumazet86cac582010-08-31 18:25:32 +00003343inline void napi_gro_flush(struct napi_struct *napi)
Herbert Xud565b0a2008-12-15 23:38:52 -08003344{
3345 struct sk_buff *skb, *next;
3346
3347 for (skb = napi->gro_list; skb; skb = next) {
3348 next = skb->next;
3349 skb->next = NULL;
3350 napi_gro_complete(skb);
3351 }
3352
Herbert Xu4ae55442009-02-08 18:00:36 +00003353 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003354 napi->gro_list = NULL;
3355}
Eric Dumazet86cac582010-08-31 18:25:32 +00003356EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003357
Ben Hutchings5b252f02009-10-29 07:17:09 +00003358enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08003359{
3360 struct sk_buff **pp = NULL;
3361 struct packet_type *ptype;
3362 __be16 type = skb->protocol;
3363 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
Herbert Xu0da2afd52008-12-26 14:57:42 -08003364 int same_flow;
Herbert Xud565b0a2008-12-15 23:38:52 -08003365 int mac_len;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003366 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003367
Jarek Poplawskice9e76c2010-08-05 01:19:11 +00003368 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
Herbert Xud565b0a2008-12-15 23:38:52 -08003369 goto normal;
3370
David S. Miller21dc3302010-08-23 00:13:46 -07003371 if (skb_is_gso(skb) || skb_has_frag_list(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08003372 goto normal;
3373
Herbert Xud565b0a2008-12-15 23:38:52 -08003374 rcu_read_lock();
3375 list_for_each_entry_rcu(ptype, head, list) {
Herbert Xud565b0a2008-12-15 23:38:52 -08003376 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3377 continue;
3378
Herbert Xu86911732009-01-29 14:19:50 +00003379 skb_set_network_header(skb, skb_gro_offset(skb));
Herbert Xud565b0a2008-12-15 23:38:52 -08003380 mac_len = skb->network_header - skb->mac_header;
3381 skb->mac_len = mac_len;
3382 NAPI_GRO_CB(skb)->same_flow = 0;
3383 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08003384 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003385
Herbert Xud565b0a2008-12-15 23:38:52 -08003386 pp = ptype->gro_receive(&napi->gro_list, skb);
3387 break;
3388 }
3389 rcu_read_unlock();
3390
3391 if (&ptype->list == head)
3392 goto normal;
3393
Herbert Xu0da2afd52008-12-26 14:57:42 -08003394 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003395 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003396
Herbert Xud565b0a2008-12-15 23:38:52 -08003397 if (pp) {
3398 struct sk_buff *nskb = *pp;
3399
3400 *pp = nskb->next;
3401 nskb->next = NULL;
3402 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00003403 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08003404 }
3405
Herbert Xu0da2afd52008-12-26 14:57:42 -08003406 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08003407 goto ok;
3408
Herbert Xu4ae55442009-02-08 18:00:36 +00003409 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08003410 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08003411
Herbert Xu4ae55442009-02-08 18:00:36 +00003412 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08003413 NAPI_GRO_CB(skb)->count = 1;
Herbert Xu86911732009-01-29 14:19:50 +00003414 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003415 skb->next = napi->gro_list;
3416 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003417 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08003418
Herbert Xuad0f9902009-02-01 01:24:55 -08003419pull:
Herbert Xucb189782009-05-26 18:50:31 +00003420 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3421 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3422
3423 BUG_ON(skb->end - skb->tail < grow);
3424
3425 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3426
3427 skb->tail += grow;
3428 skb->data_len -= grow;
3429
3430 skb_shinfo(skb)->frags[0].page_offset += grow;
3431 skb_shinfo(skb)->frags[0].size -= grow;
3432
3433 if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
3434 put_page(skb_shinfo(skb)->frags[0].page);
3435 memmove(skb_shinfo(skb)->frags,
3436 skb_shinfo(skb)->frags + 1,
Jarek Poplawskie5093ae2010-08-11 02:02:10 +00003437 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
Herbert Xucb189782009-05-26 18:50:31 +00003438 }
Herbert Xuad0f9902009-02-01 01:24:55 -08003439 }
3440
Herbert Xud565b0a2008-12-15 23:38:52 -08003441ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003442 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003443
3444normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08003445 ret = GRO_NORMAL;
3446 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08003447}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003448EXPORT_SYMBOL(dev_gro_receive);
3449
Eric Dumazet40d08022010-08-26 22:03:08 -07003450static inline gro_result_t
Ben Hutchings5b252f02009-10-29 07:17:09 +00003451__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003452{
3453 struct sk_buff *p;
3454
3455 for (p = napi->gro_list; p; p = p->next) {
Eric Dumazet40d08022010-08-26 22:03:08 -07003456 unsigned long diffs;
3457
3458 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
Jesse Gross3701e512010-10-20 13:56:06 +00003459 diffs |= p->vlan_tci ^ skb->vlan_tci;
Eric Dumazet40d08022010-08-26 22:03:08 -07003460 diffs |= compare_ether_header(skb_mac_header(p),
Joe Perchesf64f9e72009-11-29 16:55:45 -08003461 skb_gro_mac_header(skb));
Eric Dumazet40d08022010-08-26 22:03:08 -07003462 NAPI_GRO_CB(p)->same_flow = !diffs;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003463 NAPI_GRO_CB(p)->flush = 0;
3464 }
3465
3466 return dev_gro_receive(napi, skb);
3467}
Herbert Xu5d38a072009-01-04 16:13:40 -08003468
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003469gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08003470{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003471 switch (ret) {
3472 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003473 if (netif_receive_skb(skb))
3474 ret = GRO_DROP;
3475 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003476
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003477 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003478 case GRO_MERGED_FREE:
Herbert Xu5d38a072009-01-04 16:13:40 -08003479 kfree_skb(skb);
3480 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003481
3482 case GRO_HELD:
3483 case GRO_MERGED:
3484 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003485 }
3486
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003487 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003488}
3489EXPORT_SYMBOL(napi_skb_finish);
3490
Herbert Xu78a478d2009-05-26 18:50:21 +00003491void skb_gro_reset_offset(struct sk_buff *skb)
3492{
3493 NAPI_GRO_CB(skb)->data_offset = 0;
3494 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00003495 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00003496
Herbert Xu78d3fd02009-05-26 18:50:23 +00003497 if (skb->mac_header == skb->tail &&
Herbert Xu74895942009-05-26 18:50:27 +00003498 !PageHighMem(skb_shinfo(skb)->frags[0].page)) {
Herbert Xu78a478d2009-05-26 18:50:21 +00003499 NAPI_GRO_CB(skb)->frag0 =
3500 page_address(skb_shinfo(skb)->frags[0].page) +
3501 skb_shinfo(skb)->frags[0].page_offset;
Herbert Xu74895942009-05-26 18:50:27 +00003502 NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
3503 }
Herbert Xu78a478d2009-05-26 18:50:21 +00003504}
3505EXPORT_SYMBOL(skb_gro_reset_offset);
3506
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003507gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003508{
Herbert Xu86911732009-01-29 14:19:50 +00003509 skb_gro_reset_offset(skb);
3510
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003511 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003512}
3513EXPORT_SYMBOL(napi_gro_receive);
3514
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00003515static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003516{
Herbert Xu96e93ea2009-01-06 10:49:34 -08003517 __skb_pull(skb, skb_headlen(skb));
3518 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00003519 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08003520 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08003521 skb->skb_iif = 0;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003522
3523 napi->skb = skb;
3524}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003525
Herbert Xu76620aa2009-04-16 02:02:07 -07003526struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08003527{
Herbert Xu5d38a072009-01-04 16:13:40 -08003528 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003529
3530 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00003531 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3532 if (skb)
3533 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003534 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003535 return skb;
3536}
Herbert Xu76620aa2009-04-16 02:02:07 -07003537EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003538
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003539gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3540 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003541{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003542 switch (ret) {
3543 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003544 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003545 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003546
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003547 if (ret == GRO_HELD)
3548 skb_gro_pull(skb, -ETH_HLEN);
3549 else if (netif_receive_skb(skb))
3550 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003551 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003552
3553 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003554 case GRO_MERGED_FREE:
3555 napi_reuse_skb(napi, skb);
3556 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003557
3558 case GRO_MERGED:
3559 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003560 }
3561
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003562 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003563}
3564EXPORT_SYMBOL(napi_frags_finish);
3565
Herbert Xu76620aa2009-04-16 02:02:07 -07003566struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003567{
Herbert Xu76620aa2009-04-16 02:02:07 -07003568 struct sk_buff *skb = napi->skb;
3569 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00003570 unsigned int hlen;
3571 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07003572
3573 napi->skb = NULL;
3574
3575 skb_reset_mac_header(skb);
3576 skb_gro_reset_offset(skb);
3577
Herbert Xua5b1cf22009-05-26 18:50:28 +00003578 off = skb_gro_offset(skb);
3579 hlen = off + sizeof(*eth);
3580 eth = skb_gro_header_fast(skb, off);
3581 if (skb_gro_header_hard(skb, hlen)) {
3582 eth = skb_gro_header_slow(skb, hlen, off);
3583 if (unlikely(!eth)) {
3584 napi_reuse_skb(napi, skb);
3585 skb = NULL;
3586 goto out;
3587 }
Herbert Xu76620aa2009-04-16 02:02:07 -07003588 }
3589
3590 skb_gro_pull(skb, sizeof(*eth));
3591
3592 /*
3593 * This works because the only protocols we care about don't require
3594 * special handling. We'll fix it up properly at the end.
3595 */
3596 skb->protocol = eth->h_proto;
3597
3598out:
3599 return skb;
3600}
3601EXPORT_SYMBOL(napi_frags_skb);
3602
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003603gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07003604{
3605 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003606
3607 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003608 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003609
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003610 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08003611}
3612EXPORT_SYMBOL(napi_gro_frags);
3613
Eric Dumazete326bed2010-04-22 00:22:45 -07003614/*
3615 * net_rps_action sends any pending IPI's for rps.
3616 * Note: called with local irq disabled, but exits with local irq enabled.
3617 */
3618static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3619{
3620#ifdef CONFIG_RPS
3621 struct softnet_data *remsd = sd->rps_ipi_list;
3622
3623 if (remsd) {
3624 sd->rps_ipi_list = NULL;
3625
3626 local_irq_enable();
3627
3628 /* Send pending IPI's to kick RPS processing on remote cpus. */
3629 while (remsd) {
3630 struct softnet_data *next = remsd->rps_ipi_next;
3631
3632 if (cpu_online(remsd->cpu))
3633 __smp_call_function_single(remsd->cpu,
3634 &remsd->csd, 0);
3635 remsd = next;
3636 }
3637 } else
3638#endif
3639 local_irq_enable();
3640}
3641
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003642static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003643{
3644 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003645 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003646
Eric Dumazete326bed2010-04-22 00:22:45 -07003647#ifdef CONFIG_RPS
3648 /* Check if we have pending ipi, its better to send them now,
3649 * not waiting net_rx_action() end.
3650 */
3651 if (sd->rps_ipi_list) {
3652 local_irq_disable();
3653 net_rps_action_and_irq_enable(sd);
3654 }
3655#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003656 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07003657 local_irq_disable();
3658 while (work < quota) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003659 struct sk_buff *skb;
Changli Gao6e7676c2010-04-27 15:07:33 -07003660 unsigned int qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003661
Changli Gao6e7676c2010-04-27 15:07:33 -07003662 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07003663 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07003664 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07003665 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00003666 input_queue_head_incr(sd);
3667 if (++work >= quota) {
3668 local_irq_enable();
3669 return work;
3670 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003671 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003672
Changli Gao6e7676c2010-04-27 15:07:33 -07003673 rps_lock(sd);
3674 qlen = skb_queue_len(&sd->input_pkt_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003675 if (qlen)
Changli Gao6e7676c2010-04-27 15:07:33 -07003676 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3677 &sd->process_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003678
Changli Gao6e7676c2010-04-27 15:07:33 -07003679 if (qlen < quota - work) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003680 /*
3681 * Inline a custom version of __napi_complete().
3682 * only current cpu owns and manipulates this napi,
3683 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3684 * we can use a plain write instead of clear_bit(),
3685 * and we dont need an smp_mb() memory barrier.
3686 */
3687 list_del(&napi->poll_list);
3688 napi->state = 0;
3689
Changli Gao6e7676c2010-04-27 15:07:33 -07003690 quota = work + qlen;
3691 }
3692 rps_unlock(sd);
3693 }
3694 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003695
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003696 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003697}
3698
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003699/**
3700 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07003701 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003702 *
3703 * The entry's receive function will be scheduled to run
3704 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08003705void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003706{
3707 unsigned long flags;
3708
3709 local_irq_save(flags);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003710 ____napi_schedule(&__get_cpu_var(softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003711 local_irq_restore(flags);
3712}
3713EXPORT_SYMBOL(__napi_schedule);
3714
Herbert Xud565b0a2008-12-15 23:38:52 -08003715void __napi_complete(struct napi_struct *n)
3716{
3717 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3718 BUG_ON(n->gro_list);
3719
3720 list_del(&n->poll_list);
3721 smp_mb__before_clear_bit();
3722 clear_bit(NAPI_STATE_SCHED, &n->state);
3723}
3724EXPORT_SYMBOL(__napi_complete);
3725
3726void napi_complete(struct napi_struct *n)
3727{
3728 unsigned long flags;
3729
3730 /*
3731 * don't let napi dequeue from the cpu poll list
3732 * just in case its running on a different cpu
3733 */
3734 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3735 return;
3736
3737 napi_gro_flush(n);
3738 local_irq_save(flags);
3739 __napi_complete(n);
3740 local_irq_restore(flags);
3741}
3742EXPORT_SYMBOL(napi_complete);
3743
3744void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3745 int (*poll)(struct napi_struct *, int), int weight)
3746{
3747 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00003748 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003749 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08003750 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003751 napi->poll = poll;
3752 napi->weight = weight;
3753 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08003754 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08003755#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08003756 spin_lock_init(&napi->poll_lock);
3757 napi->poll_owner = -1;
3758#endif
3759 set_bit(NAPI_STATE_SCHED, &napi->state);
3760}
3761EXPORT_SYMBOL(netif_napi_add);
3762
3763void netif_napi_del(struct napi_struct *napi)
3764{
3765 struct sk_buff *skb, *next;
3766
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08003767 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07003768 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08003769
3770 for (skb = napi->gro_list; skb; skb = next) {
3771 next = skb->next;
3772 skb->next = NULL;
3773 kfree_skb(skb);
3774 }
3775
3776 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00003777 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003778}
3779EXPORT_SYMBOL(netif_napi_del);
3780
Linus Torvalds1da177e2005-04-16 15:20:36 -07003781static void net_rx_action(struct softirq_action *h)
3782{
Eric Dumazete326bed2010-04-22 00:22:45 -07003783 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003784 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07003785 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07003786 void *have;
3787
Linus Torvalds1da177e2005-04-16 15:20:36 -07003788 local_irq_disable();
3789
Eric Dumazete326bed2010-04-22 00:22:45 -07003790 while (!list_empty(&sd->poll_list)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003791 struct napi_struct *n;
3792 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003793
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003794 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003795 * Allow this to run for 2 jiffies since which will allow
3796 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003797 */
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003798 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003799 goto softnet_break;
3800
3801 local_irq_enable();
3802
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003803 /* Even though interrupts have been re-enabled, this
3804 * access is safe because interrupts can only add new
3805 * entries to the tail of this list, and only ->poll()
3806 * calls can remove this head entry from the list.
3807 */
Eric Dumazete326bed2010-04-22 00:22:45 -07003808 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003809
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003810 have = netpoll_poll_lock(n);
3811
3812 weight = n->weight;
3813
David S. Miller0a7606c2007-10-29 21:28:47 -07003814 /* This NAPI_STATE_SCHED test is for avoiding a race
3815 * with netpoll's poll_napi(). Only the entity which
3816 * obtains the lock and sees NAPI_STATE_SCHED set will
3817 * actually make the ->poll() call. Therefore we avoid
3818 * accidently calling ->poll() when NAPI is not scheduled.
3819 */
3820 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00003821 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07003822 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00003823 trace_napi_poll(n);
3824 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003825
3826 WARN_ON_ONCE(work > weight);
3827
3828 budget -= work;
3829
3830 local_irq_disable();
3831
3832 /* Drivers must not modify the NAPI state if they
3833 * consume the entire weight. In such cases this code
3834 * still "owns" the NAPI instance and therefore can
3835 * move the instance around on the list at-will.
3836 */
David S. Millerfed17f32008-01-07 21:00:40 -08003837 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07003838 if (unlikely(napi_disable_pending(n))) {
3839 local_irq_enable();
3840 napi_complete(n);
3841 local_irq_disable();
3842 } else
Eric Dumazete326bed2010-04-22 00:22:45 -07003843 list_move_tail(&n->poll_list, &sd->poll_list);
David S. Millerfed17f32008-01-07 21:00:40 -08003844 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003845
3846 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003847 }
3848out:
Eric Dumazete326bed2010-04-22 00:22:45 -07003849 net_rps_action_and_irq_enable(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003850
Chris Leechdb217332006-06-17 21:24:58 -07003851#ifdef CONFIG_NET_DMA
3852 /*
3853 * There may not be any more sk_buffs coming right now, so push
3854 * any pending DMA copies to hardware
3855 */
Dan Williams2ba05622009-01-06 11:38:14 -07003856 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07003857#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003858
Linus Torvalds1da177e2005-04-16 15:20:36 -07003859 return;
3860
3861softnet_break:
Changli Gaodee42872010-05-02 05:42:16 +00003862 sd->time_squeeze++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003863 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3864 goto out;
3865}
3866
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003867static gifconf_func_t *gifconf_list[NPROTO];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003868
3869/**
3870 * register_gifconf - register a SIOCGIF handler
3871 * @family: Address family
3872 * @gifconf: Function handler
3873 *
3874 * Register protocol dependent address dumping routines. The handler
3875 * that is passed must not be freed or reused until it has been replaced
3876 * by another handler.
3877 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003878int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003879{
3880 if (family >= NPROTO)
3881 return -EINVAL;
3882 gifconf_list[family] = gifconf;
3883 return 0;
3884}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003885EXPORT_SYMBOL(register_gifconf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003886
3887
3888/*
3889 * Map an interface index to its name (SIOCGIFNAME)
3890 */
3891
3892/*
3893 * We need this ioctl for efficient implementation of the
3894 * if_indextoname() function required by the IPv6 API. Without
3895 * it, we would have to search all the interfaces to find a
3896 * match. --pb
3897 */
3898
Eric W. Biederman881d9662007-09-17 11:56:21 -07003899static int dev_ifname(struct net *net, struct ifreq __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003900{
3901 struct net_device *dev;
3902 struct ifreq ifr;
3903
3904 /*
3905 * Fetch the caller's info block.
3906 */
3907
3908 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
3909 return -EFAULT;
3910
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003911 rcu_read_lock();
3912 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003913 if (!dev) {
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003914 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003915 return -ENODEV;
3916 }
3917
3918 strcpy(ifr.ifr_name, dev->name);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003919 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003920
3921 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
3922 return -EFAULT;
3923 return 0;
3924}
3925
3926/*
3927 * Perform a SIOCGIFCONF call. This structure will change
3928 * size eventually, and there is nothing I can do about it.
3929 * Thus we will need a 'compatibility mode'.
3930 */
3931
Eric W. Biederman881d9662007-09-17 11:56:21 -07003932static int dev_ifconf(struct net *net, char __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003933{
3934 struct ifconf ifc;
3935 struct net_device *dev;
3936 char __user *pos;
3937 int len;
3938 int total;
3939 int i;
3940
3941 /*
3942 * Fetch the caller's info block.
3943 */
3944
3945 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
3946 return -EFAULT;
3947
3948 pos = ifc.ifc_buf;
3949 len = ifc.ifc_len;
3950
3951 /*
3952 * Loop over the interfaces, and write an info block for each.
3953 */
3954
3955 total = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07003956 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003957 for (i = 0; i < NPROTO; i++) {
3958 if (gifconf_list[i]) {
3959 int done;
3960 if (!pos)
3961 done = gifconf_list[i](dev, NULL, 0);
3962 else
3963 done = gifconf_list[i](dev, pos + total,
3964 len - total);
3965 if (done < 0)
3966 return -EFAULT;
3967 total += done;
3968 }
3969 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003970 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003971
3972 /*
3973 * All done. Write the updated control block back to the caller.
3974 */
3975 ifc.ifc_len = total;
3976
3977 /*
3978 * Both BSD and Solaris return 0 here, so we do too.
3979 */
3980 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
3981}
3982
3983#ifdef CONFIG_PROC_FS
3984/*
3985 * This is invoked by the /proc filesystem handler to display a device
3986 * in detail.
3987 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003988void *dev_seq_start(struct seq_file *seq, loff_t *pos)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003989 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003990{
Denis V. Luneve372c412007-11-19 22:31:54 -08003991 struct net *net = seq_file_net(seq);
Pavel Emelianov7562f872007-05-03 15:13:45 -07003992 loff_t off;
3993 struct net_device *dev;
3994
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003995 rcu_read_lock();
Pavel Emelianov7562f872007-05-03 15:13:45 -07003996 if (!*pos)
3997 return SEQ_START_TOKEN;
3998
3999 off = 1;
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004000 for_each_netdev_rcu(net, dev)
Pavel Emelianov7562f872007-05-03 15:13:45 -07004001 if (off++ == *pos)
4002 return dev;
4003
4004 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004005}
4006
4007void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4008{
Eric Dumazetccf43432011-01-26 18:08:02 +00004009 struct net_device *dev = v;
4010
4011 if (v == SEQ_START_TOKEN)
4012 dev = first_net_device_rcu(seq_file_net(seq));
4013 else
4014 dev = next_net_device_rcu(dev);
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004015
Linus Torvalds1da177e2005-04-16 15:20:36 -07004016 ++*pos;
Eric Dumazetccf43432011-01-26 18:08:02 +00004017 return dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004018}
4019
4020void dev_seq_stop(struct seq_file *seq, void *v)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004021 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004022{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004023 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004024}
4025
4026static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
4027{
Eric Dumazet28172732010-07-07 14:58:56 -07004028 struct rtnl_link_stats64 temp;
4029 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004030
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +00004031 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
4032 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
Rusty Russell5a1b5892007-04-28 21:04:03 -07004033 dev->name, stats->rx_bytes, stats->rx_packets,
4034 stats->rx_errors,
4035 stats->rx_dropped + stats->rx_missed_errors,
4036 stats->rx_fifo_errors,
4037 stats->rx_length_errors + stats->rx_over_errors +
4038 stats->rx_crc_errors + stats->rx_frame_errors,
4039 stats->rx_compressed, stats->multicast,
4040 stats->tx_bytes, stats->tx_packets,
4041 stats->tx_errors, stats->tx_dropped,
4042 stats->tx_fifo_errors, stats->collisions,
4043 stats->tx_carrier_errors +
4044 stats->tx_aborted_errors +
4045 stats->tx_window_errors +
4046 stats->tx_heartbeat_errors,
4047 stats->tx_compressed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004048}
4049
4050/*
4051 * Called from the PROCfs module. This now uses the new arbitrary sized
4052 * /proc/net interface to create /proc/net/dev
4053 */
4054static int dev_seq_show(struct seq_file *seq, void *v)
4055{
4056 if (v == SEQ_START_TOKEN)
4057 seq_puts(seq, "Inter-| Receive "
4058 " | Transmit\n"
4059 " face |bytes packets errs drop fifo frame "
4060 "compressed multicast|bytes packets errs "
4061 "drop fifo colls carrier compressed\n");
4062 else
4063 dev_seq_printf_stats(seq, v);
4064 return 0;
4065}
4066
Changli Gaodee42872010-05-02 05:42:16 +00004067static struct softnet_data *softnet_get_online(loff_t *pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004068{
Changli Gaodee42872010-05-02 05:42:16 +00004069 struct softnet_data *sd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004070
Mike Travis0c0b0ac2008-05-02 16:43:08 -07004071 while (*pos < nr_cpu_ids)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004072 if (cpu_online(*pos)) {
Changli Gaodee42872010-05-02 05:42:16 +00004073 sd = &per_cpu(softnet_data, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004074 break;
4075 } else
4076 ++*pos;
Changli Gaodee42872010-05-02 05:42:16 +00004077 return sd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004078}
4079
4080static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
4081{
4082 return softnet_get_online(pos);
4083}
4084
4085static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4086{
4087 ++*pos;
4088 return softnet_get_online(pos);
4089}
4090
4091static void softnet_seq_stop(struct seq_file *seq, void *v)
4092{
4093}
4094
4095static int softnet_seq_show(struct seq_file *seq, void *v)
4096{
Changli Gaodee42872010-05-02 05:42:16 +00004097 struct softnet_data *sd = v;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004098
Tom Herbert0a9627f2010-03-16 08:03:29 +00004099 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
Changli Gaodee42872010-05-02 05:42:16 +00004100 sd->processed, sd->dropped, sd->time_squeeze, 0,
Stephen Hemmingerc1ebcdb2005-06-23 20:08:59 -07004101 0, 0, 0, 0, /* was fastroute */
Changli Gaodee42872010-05-02 05:42:16 +00004102 sd->cpu_collision, sd->received_rps);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004103 return 0;
4104}
4105
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004106static const struct seq_operations dev_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004107 .start = dev_seq_start,
4108 .next = dev_seq_next,
4109 .stop = dev_seq_stop,
4110 .show = dev_seq_show,
4111};
4112
4113static int dev_seq_open(struct inode *inode, struct file *file)
4114{
Denis V. Luneve372c412007-11-19 22:31:54 -08004115 return seq_open_net(inode, file, &dev_seq_ops,
4116 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004117}
4118
Arjan van de Ven9a321442007-02-12 00:55:35 -08004119static const struct file_operations dev_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004120 .owner = THIS_MODULE,
4121 .open = dev_seq_open,
4122 .read = seq_read,
4123 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004124 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004125};
4126
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004127static const struct seq_operations softnet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004128 .start = softnet_seq_start,
4129 .next = softnet_seq_next,
4130 .stop = softnet_seq_stop,
4131 .show = softnet_seq_show,
4132};
4133
4134static int softnet_seq_open(struct inode *inode, struct file *file)
4135{
4136 return seq_open(file, &softnet_seq_ops);
4137}
4138
Arjan van de Ven9a321442007-02-12 00:55:35 -08004139static const struct file_operations softnet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004140 .owner = THIS_MODULE,
4141 .open = softnet_seq_open,
4142 .read = seq_read,
4143 .llseek = seq_lseek,
4144 .release = seq_release,
4145};
4146
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004147static void *ptype_get_idx(loff_t pos)
4148{
4149 struct packet_type *pt = NULL;
4150 loff_t i = 0;
4151 int t;
4152
4153 list_for_each_entry_rcu(pt, &ptype_all, list) {
4154 if (i == pos)
4155 return pt;
4156 ++i;
4157 }
4158
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004159 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004160 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
4161 if (i == pos)
4162 return pt;
4163 ++i;
4164 }
4165 }
4166 return NULL;
4167}
4168
4169static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004170 __acquires(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004171{
4172 rcu_read_lock();
4173 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
4174}
4175
4176static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4177{
4178 struct packet_type *pt;
4179 struct list_head *nxt;
4180 int hash;
4181
4182 ++*pos;
4183 if (v == SEQ_START_TOKEN)
4184 return ptype_get_idx(0);
4185
4186 pt = v;
4187 nxt = pt->list.next;
4188 if (pt->type == htons(ETH_P_ALL)) {
4189 if (nxt != &ptype_all)
4190 goto found;
4191 hash = 0;
4192 nxt = ptype_base[0].next;
4193 } else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004194 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004195
4196 while (nxt == &ptype_base[hash]) {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004197 if (++hash >= PTYPE_HASH_SIZE)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004198 return NULL;
4199 nxt = ptype_base[hash].next;
4200 }
4201found:
4202 return list_entry(nxt, struct packet_type, list);
4203}
4204
4205static void ptype_seq_stop(struct seq_file *seq, void *v)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004206 __releases(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004207{
4208 rcu_read_unlock();
4209}
4210
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004211static int ptype_seq_show(struct seq_file *seq, void *v)
4212{
4213 struct packet_type *pt = v;
4214
4215 if (v == SEQ_START_TOKEN)
4216 seq_puts(seq, "Type Device Function\n");
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004217 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004218 if (pt->type == htons(ETH_P_ALL))
4219 seq_puts(seq, "ALL ");
4220 else
4221 seq_printf(seq, "%04x", ntohs(pt->type));
4222
Alexey Dobriyan908cd2d2008-11-16 19:50:35 -08004223 seq_printf(seq, " %-8s %pF\n",
4224 pt->dev ? pt->dev->name : "", pt->func);
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004225 }
4226
4227 return 0;
4228}
4229
4230static const struct seq_operations ptype_seq_ops = {
4231 .start = ptype_seq_start,
4232 .next = ptype_seq_next,
4233 .stop = ptype_seq_stop,
4234 .show = ptype_seq_show,
4235};
4236
4237static int ptype_seq_open(struct inode *inode, struct file *file)
4238{
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004239 return seq_open_net(inode, file, &ptype_seq_ops,
4240 sizeof(struct seq_net_private));
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004241}
4242
4243static const struct file_operations ptype_seq_fops = {
4244 .owner = THIS_MODULE,
4245 .open = ptype_seq_open,
4246 .read = seq_read,
4247 .llseek = seq_lseek,
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004248 .release = seq_release_net,
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004249};
4250
4251
Pavel Emelyanov46650792007-10-08 20:38:39 -07004252static int __net_init dev_proc_net_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004253{
4254 int rc = -ENOMEM;
4255
Eric W. Biederman881d9662007-09-17 11:56:21 -07004256 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004257 goto out;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004258 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004259 goto out_dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004260 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004261 goto out_softnet;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004262
Eric W. Biederman881d9662007-09-17 11:56:21 -07004263 if (wext_proc_init(net))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004264 goto out_ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004265 rc = 0;
4266out:
4267 return rc;
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004268out_ptype:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004269 proc_net_remove(net, "ptype");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004270out_softnet:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004271 proc_net_remove(net, "softnet_stat");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004272out_dev:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004273 proc_net_remove(net, "dev");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004274 goto out;
4275}
Eric W. Biederman881d9662007-09-17 11:56:21 -07004276
Pavel Emelyanov46650792007-10-08 20:38:39 -07004277static void __net_exit dev_proc_net_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004278{
4279 wext_proc_exit(net);
4280
4281 proc_net_remove(net, "ptype");
4282 proc_net_remove(net, "softnet_stat");
4283 proc_net_remove(net, "dev");
4284}
4285
Denis V. Lunev022cbae2007-11-13 03:23:50 -08004286static struct pernet_operations __net_initdata dev_proc_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07004287 .init = dev_proc_net_init,
4288 .exit = dev_proc_net_exit,
4289};
4290
4291static int __init dev_proc_init(void)
4292{
4293 return register_pernet_subsys(&dev_proc_ops);
4294}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004295#else
4296#define dev_proc_init() 0
4297#endif /* CONFIG_PROC_FS */
4298
4299
4300/**
Jiri Pirko1765a572011-02-12 06:48:36 +00004301 * netdev_set_master - set up master pointer
Linus Torvalds1da177e2005-04-16 15:20:36 -07004302 * @slave: slave device
4303 * @master: new master device
4304 *
4305 * Changes the master device of the slave. Pass %NULL to break the
4306 * bonding. The caller must hold the RTNL semaphore. On a failure
4307 * a negative errno code is returned. On success the reference counts
Jiri Pirko1765a572011-02-12 06:48:36 +00004308 * are adjusted and the function returns zero.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004309 */
4310int netdev_set_master(struct net_device *slave, struct net_device *master)
4311{
4312 struct net_device *old = slave->master;
4313
4314 ASSERT_RTNL();
4315
4316 if (master) {
4317 if (old)
4318 return -EBUSY;
4319 dev_hold(master);
4320 }
4321
4322 slave->master = master;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004323
Eric Dumazet283f2fe2010-03-18 13:37:40 +00004324 if (old) {
4325 synchronize_net();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004326 dev_put(old);
Eric Dumazet283f2fe2010-03-18 13:37:40 +00004327 }
Jiri Pirko1765a572011-02-12 06:48:36 +00004328 return 0;
4329}
4330EXPORT_SYMBOL(netdev_set_master);
4331
4332/**
4333 * netdev_set_bond_master - set up bonding master/slave pair
4334 * @slave: slave device
4335 * @master: new master device
4336 *
4337 * Changes the master device of the slave. Pass %NULL to break the
4338 * bonding. The caller must hold the RTNL semaphore. On a failure
4339 * a negative errno code is returned. On success %RTM_NEWLINK is sent
4340 * to the routing socket and the function returns zero.
4341 */
4342int netdev_set_bond_master(struct net_device *slave, struct net_device *master)
4343{
4344 int err;
4345
4346 ASSERT_RTNL();
4347
4348 err = netdev_set_master(slave, master);
4349 if (err)
4350 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004351 if (master)
4352 slave->flags |= IFF_SLAVE;
4353 else
4354 slave->flags &= ~IFF_SLAVE;
4355
4356 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4357 return 0;
4358}
Jiri Pirko1765a572011-02-12 06:48:36 +00004359EXPORT_SYMBOL(netdev_set_bond_master);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004360
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004361static void dev_change_rx_flags(struct net_device *dev, int flags)
4362{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004363 const struct net_device_ops *ops = dev->netdev_ops;
4364
4365 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4366 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004367}
4368
Wang Chendad9b332008-06-18 01:48:28 -07004369static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07004370{
4371 unsigned short old_flags = dev->flags;
David Howells8192b0c2008-11-14 10:39:10 +11004372 uid_t uid;
4373 gid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07004374
Patrick McHardy24023452007-07-14 18:51:31 -07004375 ASSERT_RTNL();
4376
Wang Chendad9b332008-06-18 01:48:28 -07004377 dev->flags |= IFF_PROMISC;
4378 dev->promiscuity += inc;
4379 if (dev->promiscuity == 0) {
4380 /*
4381 * Avoid overflow.
4382 * If inc causes overflow, untouch promisc and return error.
4383 */
4384 if (inc < 0)
4385 dev->flags &= ~IFF_PROMISC;
4386 else {
4387 dev->promiscuity -= inc;
4388 printk(KERN_WARNING "%s: promiscuity touches roof, "
4389 "set promiscuity failed, promiscuity feature "
4390 "of device might be broken.\n", dev->name);
4391 return -EOVERFLOW;
4392 }
4393 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004394 if (dev->flags != old_flags) {
4395 printk(KERN_INFO "device %s %s promiscuous mode\n",
4396 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
4397 "left");
David Howells8192b0c2008-11-14 10:39:10 +11004398 if (audit_enabled) {
4399 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004400 audit_log(current->audit_context, GFP_ATOMIC,
4401 AUDIT_ANOM_PROMISCUOUS,
4402 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4403 dev->name, (dev->flags & IFF_PROMISC),
4404 (old_flags & IFF_PROMISC),
4405 audit_get_loginuid(current),
David Howells8192b0c2008-11-14 10:39:10 +11004406 uid, gid,
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004407 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11004408 }
Patrick McHardy24023452007-07-14 18:51:31 -07004409
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004410 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07004411 }
Wang Chendad9b332008-06-18 01:48:28 -07004412 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004413}
4414
Linus Torvalds1da177e2005-04-16 15:20:36 -07004415/**
4416 * dev_set_promiscuity - update promiscuity count on a device
4417 * @dev: device
4418 * @inc: modifier
4419 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07004420 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004421 * remains above zero the interface remains promiscuous. Once it hits zero
4422 * the device reverts back to normal filtering operation. A negative inc
4423 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07004424 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004425 */
Wang Chendad9b332008-06-18 01:48:28 -07004426int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004427{
4428 unsigned short old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07004429 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004430
Wang Chendad9b332008-06-18 01:48:28 -07004431 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07004432 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07004433 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07004434 if (dev->flags != old_flags)
4435 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07004436 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004437}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004438EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004439
4440/**
4441 * dev_set_allmulti - update allmulti count on a device
4442 * @dev: device
4443 * @inc: modifier
4444 *
4445 * Add or remove reception of all multicast frames to a device. While the
4446 * count in the device remains above zero the interface remains listening
4447 * to all interfaces. Once it hits zero the device reverts back to normal
4448 * filtering operation. A negative @inc value is used to drop the counter
4449 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07004450 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004451 */
4452
Wang Chendad9b332008-06-18 01:48:28 -07004453int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004454{
4455 unsigned short old_flags = dev->flags;
4456
Patrick McHardy24023452007-07-14 18:51:31 -07004457 ASSERT_RTNL();
4458
Linus Torvalds1da177e2005-04-16 15:20:36 -07004459 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07004460 dev->allmulti += inc;
4461 if (dev->allmulti == 0) {
4462 /*
4463 * Avoid overflow.
4464 * If inc causes overflow, untouch allmulti and return error.
4465 */
4466 if (inc < 0)
4467 dev->flags &= ~IFF_ALLMULTI;
4468 else {
4469 dev->allmulti -= inc;
4470 printk(KERN_WARNING "%s: allmulti touches roof, "
4471 "set allmulti failed, allmulti feature of "
4472 "device might be broken.\n", dev->name);
4473 return -EOVERFLOW;
4474 }
4475 }
Patrick McHardy24023452007-07-14 18:51:31 -07004476 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004477 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07004478 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07004479 }
Wang Chendad9b332008-06-18 01:48:28 -07004480 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004481}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004482EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07004483
4484/*
4485 * Upload unicast and multicast address lists to device and
4486 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08004487 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07004488 * are present.
4489 */
4490void __dev_set_rx_mode(struct net_device *dev)
4491{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004492 const struct net_device_ops *ops = dev->netdev_ops;
4493
Patrick McHardy4417da62007-06-27 01:28:10 -07004494 /* dev_open will call this function so the list will stay sane. */
4495 if (!(dev->flags&IFF_UP))
4496 return;
4497
4498 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09004499 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07004500
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004501 if (ops->ndo_set_rx_mode)
4502 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004503 else {
4504 /* Unicast addresses changes may only happen under the rtnl,
4505 * therefore calling __dev_set_promiscuity here is safe.
4506 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004507 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004508 __dev_set_promiscuity(dev, 1);
4509 dev->uc_promisc = 1;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004510 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004511 __dev_set_promiscuity(dev, -1);
4512 dev->uc_promisc = 0;
4513 }
4514
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004515 if (ops->ndo_set_multicast_list)
4516 ops->ndo_set_multicast_list(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004517 }
4518}
4519
4520void dev_set_rx_mode(struct net_device *dev)
4521{
David S. Millerb9e40852008-07-15 00:15:08 -07004522 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004523 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07004524 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004525}
4526
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004527/**
4528 * dev_get_flags - get flags reported to userspace
4529 * @dev: device
4530 *
4531 * Get the combination of flag bits exported through APIs to userspace.
4532 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004533unsigned dev_get_flags(const struct net_device *dev)
4534{
4535 unsigned flags;
4536
4537 flags = (dev->flags & ~(IFF_PROMISC |
4538 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004539 IFF_RUNNING |
4540 IFF_LOWER_UP |
4541 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004542 (dev->gflags & (IFF_PROMISC |
4543 IFF_ALLMULTI));
4544
Stefan Rompfb00055a2006-03-20 17:09:11 -08004545 if (netif_running(dev)) {
4546 if (netif_oper_up(dev))
4547 flags |= IFF_RUNNING;
4548 if (netif_carrier_ok(dev))
4549 flags |= IFF_LOWER_UP;
4550 if (netif_dormant(dev))
4551 flags |= IFF_DORMANT;
4552 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004553
4554 return flags;
4555}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004556EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004557
Patrick McHardybd380812010-02-26 06:34:53 +00004558int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004559{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004560 int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004561 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004562
Patrick McHardy24023452007-07-14 18:51:31 -07004563 ASSERT_RTNL();
4564
Linus Torvalds1da177e2005-04-16 15:20:36 -07004565 /*
4566 * Set the flags on our device.
4567 */
4568
4569 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4570 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4571 IFF_AUTOMEDIA)) |
4572 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4573 IFF_ALLMULTI));
4574
4575 /*
4576 * Load in the correct multicast list now the flags have changed.
4577 */
4578
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004579 if ((old_flags ^ flags) & IFF_MULTICAST)
4580 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07004581
Patrick McHardy4417da62007-06-27 01:28:10 -07004582 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004583
4584 /*
4585 * Have we downed the interface. We handle IFF_UP ourselves
4586 * according to user attempts to set it, rather than blindly
4587 * setting it.
4588 */
4589
4590 ret = 0;
4591 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00004592 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004593
4594 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07004595 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004596 }
4597
Linus Torvalds1da177e2005-04-16 15:20:36 -07004598 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004599 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4600
Linus Torvalds1da177e2005-04-16 15:20:36 -07004601 dev->gflags ^= IFF_PROMISC;
4602 dev_set_promiscuity(dev, inc);
4603 }
4604
4605 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4606 is important. Some (broken) drivers set IFF_PROMISC, when
4607 IFF_ALLMULTI is requested not asking us and not reporting.
4608 */
4609 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004610 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4611
Linus Torvalds1da177e2005-04-16 15:20:36 -07004612 dev->gflags ^= IFF_ALLMULTI;
4613 dev_set_allmulti(dev, inc);
4614 }
4615
Patrick McHardybd380812010-02-26 06:34:53 +00004616 return ret;
4617}
4618
4619void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4620{
4621 unsigned int changes = dev->flags ^ old_flags;
4622
4623 if (changes & IFF_UP) {
4624 if (dev->flags & IFF_UP)
4625 call_netdevice_notifiers(NETDEV_UP, dev);
4626 else
4627 call_netdevice_notifiers(NETDEV_DOWN, dev);
4628 }
4629
4630 if (dev->flags & IFF_UP &&
4631 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4632 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4633}
4634
4635/**
4636 * dev_change_flags - change device settings
4637 * @dev: device
4638 * @flags: device state flags
4639 *
4640 * Change settings on device based state flags. The flags are
4641 * in the userspace exported format.
4642 */
4643int dev_change_flags(struct net_device *dev, unsigned flags)
4644{
4645 int ret, changes;
4646 int old_flags = dev->flags;
4647
4648 ret = __dev_change_flags(dev, flags);
4649 if (ret < 0)
4650 return ret;
4651
4652 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07004653 if (changes)
4654 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004655
Patrick McHardybd380812010-02-26 06:34:53 +00004656 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004657 return ret;
4658}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004659EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004660
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004661/**
4662 * dev_set_mtu - Change maximum transfer unit
4663 * @dev: device
4664 * @new_mtu: new transfer unit
4665 *
4666 * Change the maximum transfer size of the network device.
4667 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004668int dev_set_mtu(struct net_device *dev, int new_mtu)
4669{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004670 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004671 int err;
4672
4673 if (new_mtu == dev->mtu)
4674 return 0;
4675
4676 /* MTU must be positive. */
4677 if (new_mtu < 0)
4678 return -EINVAL;
4679
4680 if (!netif_device_present(dev))
4681 return -ENODEV;
4682
4683 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004684 if (ops->ndo_change_mtu)
4685 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004686 else
4687 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004688
Linus Torvalds1da177e2005-04-16 15:20:36 -07004689 if (!err && dev->flags & IFF_UP)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004690 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004691 return err;
4692}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004693EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004694
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004695/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00004696 * dev_set_group - Change group this device belongs to
4697 * @dev: device
4698 * @new_group: group this device should belong to
4699 */
4700void dev_set_group(struct net_device *dev, int new_group)
4701{
4702 dev->group = new_group;
4703}
4704EXPORT_SYMBOL(dev_set_group);
4705
4706/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004707 * dev_set_mac_address - Change Media Access Control Address
4708 * @dev: device
4709 * @sa: new address
4710 *
4711 * Change the hardware (MAC) address of the device
4712 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004713int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4714{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004715 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004716 int err;
4717
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004718 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004719 return -EOPNOTSUPP;
4720 if (sa->sa_family != dev->type)
4721 return -EINVAL;
4722 if (!netif_device_present(dev))
4723 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004724 err = ops->ndo_set_mac_address(dev, sa);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004725 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004726 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004727 return err;
4728}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004729EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004730
4731/*
Eric Dumazet3710bec2009-11-01 19:42:09 +00004732 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07004733 */
Jeff Garzik14e3e072007-10-08 00:06:32 -07004734static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004735{
4736 int err;
Eric Dumazet3710bec2009-11-01 19:42:09 +00004737 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004738
4739 if (!dev)
4740 return -ENODEV;
4741
4742 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004743 case SIOCGIFFLAGS: /* Get interface flags */
4744 ifr->ifr_flags = (short) dev_get_flags(dev);
4745 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004746
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004747 case SIOCGIFMETRIC: /* Get the metric on the interface
4748 (currently unused) */
4749 ifr->ifr_metric = 0;
4750 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004751
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004752 case SIOCGIFMTU: /* Get the MTU of a device */
4753 ifr->ifr_mtu = dev->mtu;
4754 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004755
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004756 case SIOCGIFHWADDR:
4757 if (!dev->addr_len)
4758 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4759 else
4760 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4761 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4762 ifr->ifr_hwaddr.sa_family = dev->type;
4763 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004764
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004765 case SIOCGIFSLAVE:
4766 err = -EINVAL;
4767 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004768
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004769 case SIOCGIFMAP:
4770 ifr->ifr_map.mem_start = dev->mem_start;
4771 ifr->ifr_map.mem_end = dev->mem_end;
4772 ifr->ifr_map.base_addr = dev->base_addr;
4773 ifr->ifr_map.irq = dev->irq;
4774 ifr->ifr_map.dma = dev->dma;
4775 ifr->ifr_map.port = dev->if_port;
4776 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004777
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004778 case SIOCGIFINDEX:
4779 ifr->ifr_ifindex = dev->ifindex;
4780 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004781
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004782 case SIOCGIFTXQLEN:
4783 ifr->ifr_qlen = dev->tx_queue_len;
4784 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004785
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004786 default:
4787 /* dev_ioctl() should ensure this case
4788 * is never reached
4789 */
4790 WARN_ON(1);
4791 err = -EINVAL;
4792 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004793
4794 }
4795 return err;
4796}
4797
4798/*
4799 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4800 */
4801static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4802{
4803 int err;
4804 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004805 const struct net_device_ops *ops;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004806
4807 if (!dev)
4808 return -ENODEV;
4809
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004810 ops = dev->netdev_ops;
4811
Jeff Garzik14e3e072007-10-08 00:06:32 -07004812 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004813 case SIOCSIFFLAGS: /* Set interface flags */
4814 return dev_change_flags(dev, ifr->ifr_flags);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004815
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004816 case SIOCSIFMETRIC: /* Set the metric on the interface
4817 (currently unused) */
4818 return -EOPNOTSUPP;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004819
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004820 case SIOCSIFMTU: /* Set the MTU of a device */
4821 return dev_set_mtu(dev, ifr->ifr_mtu);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004822
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004823 case SIOCSIFHWADDR:
4824 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004825
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004826 case SIOCSIFHWBROADCAST:
4827 if (ifr->ifr_hwaddr.sa_family != dev->type)
4828 return -EINVAL;
4829 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4830 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4831 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4832 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004833
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004834 case SIOCSIFMAP:
4835 if (ops->ndo_set_config) {
4836 if (!netif_device_present(dev))
4837 return -ENODEV;
4838 return ops->ndo_set_config(dev, &ifr->ifr_map);
4839 }
4840 return -EOPNOTSUPP;
4841
4842 case SIOCADDMULTI:
4843 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4844 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4845 return -EINVAL;
4846 if (!netif_device_present(dev))
4847 return -ENODEV;
Jiri Pirko22bedad32010-04-01 21:22:57 +00004848 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004849
4850 case SIOCDELMULTI:
4851 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4852 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4853 return -EINVAL;
4854 if (!netif_device_present(dev))
4855 return -ENODEV;
Jiri Pirko22bedad32010-04-01 21:22:57 +00004856 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004857
4858 case SIOCSIFTXQLEN:
4859 if (ifr->ifr_qlen < 0)
4860 return -EINVAL;
4861 dev->tx_queue_len = ifr->ifr_qlen;
4862 return 0;
4863
4864 case SIOCSIFNAME:
4865 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4866 return dev_change_name(dev, ifr->ifr_newname);
4867
4868 /*
4869 * Unknown or private ioctl
4870 */
4871 default:
4872 if ((cmd >= SIOCDEVPRIVATE &&
4873 cmd <= SIOCDEVPRIVATE + 15) ||
4874 cmd == SIOCBONDENSLAVE ||
4875 cmd == SIOCBONDRELEASE ||
4876 cmd == SIOCBONDSETHWADDR ||
4877 cmd == SIOCBONDSLAVEINFOQUERY ||
4878 cmd == SIOCBONDINFOQUERY ||
4879 cmd == SIOCBONDCHANGEACTIVE ||
4880 cmd == SIOCGMIIPHY ||
4881 cmd == SIOCGMIIREG ||
4882 cmd == SIOCSMIIREG ||
4883 cmd == SIOCBRADDIF ||
4884 cmd == SIOCBRDELIF ||
4885 cmd == SIOCSHWTSTAMP ||
4886 cmd == SIOCWANDEV) {
4887 err = -EOPNOTSUPP;
4888 if (ops->ndo_do_ioctl) {
4889 if (netif_device_present(dev))
4890 err = ops->ndo_do_ioctl(dev, ifr, cmd);
4891 else
4892 err = -ENODEV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004893 }
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004894 } else
4895 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004896
4897 }
4898 return err;
4899}
4900
4901/*
4902 * This function handles all "interface"-type I/O control requests. The actual
4903 * 'doing' part of this is dev_ifsioc above.
4904 */
4905
4906/**
4907 * dev_ioctl - network device ioctl
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004908 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07004909 * @cmd: command to issue
4910 * @arg: pointer to a struct ifreq in user space
4911 *
4912 * Issue ioctl functions to devices. This is normally called by the
4913 * user space syscall interfaces but can sometimes be useful for
4914 * other purposes. The return value is the return from the syscall if
4915 * positive or a negative errno code on error.
4916 */
4917
Eric W. Biederman881d9662007-09-17 11:56:21 -07004918int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004919{
4920 struct ifreq ifr;
4921 int ret;
4922 char *colon;
4923
4924 /* One special case: SIOCGIFCONF takes ifconf argument
4925 and requires shared lock, because it sleeps writing
4926 to user space.
4927 */
4928
4929 if (cmd == SIOCGIFCONF) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004930 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07004931 ret = dev_ifconf(net, (char __user *) arg);
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004932 rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004933 return ret;
4934 }
4935 if (cmd == SIOCGIFNAME)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004936 return dev_ifname(net, (struct ifreq __user *)arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004937
4938 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4939 return -EFAULT;
4940
4941 ifr.ifr_name[IFNAMSIZ-1] = 0;
4942
4943 colon = strchr(ifr.ifr_name, ':');
4944 if (colon)
4945 *colon = 0;
4946
4947 /*
4948 * See which interface the caller is talking about.
4949 */
4950
4951 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004952 /*
4953 * These ioctl calls:
4954 * - can be done by all.
4955 * - atomic and do not require locking.
4956 * - return a value
4957 */
4958 case SIOCGIFFLAGS:
4959 case SIOCGIFMETRIC:
4960 case SIOCGIFMTU:
4961 case SIOCGIFHWADDR:
4962 case SIOCGIFSLAVE:
4963 case SIOCGIFMAP:
4964 case SIOCGIFINDEX:
4965 case SIOCGIFTXQLEN:
4966 dev_load(net, ifr.ifr_name);
Eric Dumazet3710bec2009-11-01 19:42:09 +00004967 rcu_read_lock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004968 ret = dev_ifsioc_locked(net, &ifr, cmd);
Eric Dumazet3710bec2009-11-01 19:42:09 +00004969 rcu_read_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004970 if (!ret) {
4971 if (colon)
4972 *colon = ':';
4973 if (copy_to_user(arg, &ifr,
4974 sizeof(struct ifreq)))
4975 ret = -EFAULT;
4976 }
4977 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004978
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004979 case SIOCETHTOOL:
4980 dev_load(net, ifr.ifr_name);
4981 rtnl_lock();
4982 ret = dev_ethtool(net, &ifr);
4983 rtnl_unlock();
4984 if (!ret) {
4985 if (colon)
4986 *colon = ':';
4987 if (copy_to_user(arg, &ifr,
4988 sizeof(struct ifreq)))
4989 ret = -EFAULT;
4990 }
4991 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004992
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004993 /*
4994 * These ioctl calls:
4995 * - require superuser power.
4996 * - require strict serialization.
4997 * - return a value
4998 */
4999 case SIOCGMIIPHY:
5000 case SIOCGMIIREG:
5001 case SIOCSIFNAME:
5002 if (!capable(CAP_NET_ADMIN))
5003 return -EPERM;
5004 dev_load(net, ifr.ifr_name);
5005 rtnl_lock();
5006 ret = dev_ifsioc(net, &ifr, cmd);
5007 rtnl_unlock();
5008 if (!ret) {
5009 if (colon)
5010 *colon = ':';
5011 if (copy_to_user(arg, &ifr,
5012 sizeof(struct ifreq)))
5013 ret = -EFAULT;
5014 }
5015 return ret;
5016
5017 /*
5018 * These ioctl calls:
5019 * - require superuser power.
5020 * - require strict serialization.
5021 * - do not return a value
5022 */
5023 case SIOCSIFFLAGS:
5024 case SIOCSIFMETRIC:
5025 case SIOCSIFMTU:
5026 case SIOCSIFMAP:
5027 case SIOCSIFHWADDR:
5028 case SIOCSIFSLAVE:
5029 case SIOCADDMULTI:
5030 case SIOCDELMULTI:
5031 case SIOCSIFHWBROADCAST:
5032 case SIOCSIFTXQLEN:
5033 case SIOCSMIIREG:
5034 case SIOCBONDENSLAVE:
5035 case SIOCBONDRELEASE:
5036 case SIOCBONDSETHWADDR:
5037 case SIOCBONDCHANGEACTIVE:
5038 case SIOCBRADDIF:
5039 case SIOCBRDELIF:
5040 case SIOCSHWTSTAMP:
5041 if (!capable(CAP_NET_ADMIN))
5042 return -EPERM;
5043 /* fall through */
5044 case SIOCBONDSLAVEINFOQUERY:
5045 case SIOCBONDINFOQUERY:
5046 dev_load(net, ifr.ifr_name);
5047 rtnl_lock();
5048 ret = dev_ifsioc(net, &ifr, cmd);
5049 rtnl_unlock();
5050 return ret;
5051
5052 case SIOCGIFMEM:
5053 /* Get the per device memory space. We can add this but
5054 * currently do not support it */
5055 case SIOCSIFMEM:
5056 /* Set the per device memory buffer space.
5057 * Not applicable in our case */
5058 case SIOCSIFLINK:
5059 return -EINVAL;
5060
5061 /*
5062 * Unknown or private ioctl.
5063 */
5064 default:
5065 if (cmd == SIOCWANDEV ||
5066 (cmd >= SIOCDEVPRIVATE &&
5067 cmd <= SIOCDEVPRIVATE + 15)) {
Eric W. Biederman881d9662007-09-17 11:56:21 -07005068 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005069 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005070 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005071 rtnl_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005072 if (!ret && copy_to_user(arg, &ifr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005073 sizeof(struct ifreq)))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005074 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005075 return ret;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005076 }
5077 /* Take care of Wireless Extensions */
5078 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
5079 return wext_handle_ioctl(net, &ifr, cmd, arg);
5080 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005081 }
5082}
5083
5084
5085/**
5086 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005087 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005088 *
5089 * Returns a suitable unique value for a new device interface
5090 * number. The caller must hold the rtnl semaphore or the
5091 * dev_base_lock to be sure it remains unique.
5092 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07005093static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005094{
5095 static int ifindex;
5096 for (;;) {
5097 if (++ifindex <= 0)
5098 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005099 if (!__dev_get_by_index(net, ifindex))
Linus Torvalds1da177e2005-04-16 15:20:36 -07005100 return ifindex;
5101 }
5102}
5103
Linus Torvalds1da177e2005-04-16 15:20:36 -07005104/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08005105static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005106
Stephen Hemminger6f05f622007-03-08 20:46:03 -08005107static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005108{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005109 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005110}
5111
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005112static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005113{
Krishna Kumare93737b2009-12-08 22:26:02 +00005114 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005115
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005116 BUG_ON(dev_boot_phase);
5117 ASSERT_RTNL();
5118
Krishna Kumare93737b2009-12-08 22:26:02 +00005119 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005120 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00005121 * for initialization unwind. Remove those
5122 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005123 */
5124 if (dev->reg_state == NETREG_UNINITIALIZED) {
5125 pr_debug("unregister_netdevice: device %s/%p never "
5126 "was registered\n", dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005127
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005128 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00005129 list_del(&dev->unreg_list);
5130 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005131 }
5132
5133 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00005134 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005135
Octavian Purdila44345722010-12-13 12:44:07 +00005136 /* If device is running, close it first. */
5137 dev_close_many(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005138
Octavian Purdila44345722010-12-13 12:44:07 +00005139 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005140 /* And unlink it from device chain. */
5141 unlist_netdevice(dev);
5142
5143 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005144 }
5145
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005146 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005147
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005148 list_for_each_entry(dev, head, unreg_list) {
5149 /* Shutdown queueing discipline. */
5150 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005151
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005152
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005153 /* Notify protocols, that we are about to destroy
5154 this device. They should clean all the things.
5155 */
5156 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5157
Patrick McHardya2835762010-02-26 06:34:51 +00005158 if (!dev->rtnl_link_ops ||
5159 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5160 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5161
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005162 /*
5163 * Flush the unicast and multicast chains
5164 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005165 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00005166 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005167
5168 if (dev->netdev_ops->ndo_uninit)
5169 dev->netdev_ops->ndo_uninit(dev);
5170
5171 /* Notifier chain MUST detach us from master device. */
5172 WARN_ON(dev->master);
5173
5174 /* Remove entries from kobject tree */
5175 netdev_unregister_kobject(dev);
5176 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005177
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005178 /* Process any work delayed until the end of the batch */
stephen hemmingere5e26d72010-02-24 14:01:38 +00005179 dev = list_first_entry(head, struct net_device, unreg_list);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005180 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5181
Eric Dumazetef885af2010-09-13 12:24:54 +00005182 rcu_barrier();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005183
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005184 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005185 dev_put(dev);
5186}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005187
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005188static void rollback_registered(struct net_device *dev)
5189{
5190 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005191
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005192 list_add(&dev->unreg_list, &single);
5193 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00005194 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005195}
5196
Michał Mirosławacd11302011-01-24 15:45:15 -08005197u32 netdev_fix_features(struct net_device *dev, u32 features)
Herbert Xub63365a2008-10-23 01:11:29 -07005198{
Michał Mirosław57422dc2011-01-22 12:14:12 +00005199 /* Fix illegal checksum combinations */
5200 if ((features & NETIF_F_HW_CSUM) &&
5201 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosławacd11302011-01-24 15:45:15 -08005202 netdev_info(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005203 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5204 }
5205
5206 if ((features & NETIF_F_NO_CSUM) &&
5207 (features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosławacd11302011-01-24 15:45:15 -08005208 netdev_info(dev, "mixed no checksumming and other settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005209 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5210 }
5211
Herbert Xub63365a2008-10-23 01:11:29 -07005212 /* Fix illegal SG+CSUM combinations. */
5213 if ((features & NETIF_F_SG) &&
5214 !(features & NETIF_F_ALL_CSUM)) {
Michał Mirosławacd11302011-01-24 15:45:15 -08005215 netdev_info(dev,
5216 "Dropping NETIF_F_SG since no checksum feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005217 features &= ~NETIF_F_SG;
5218 }
5219
5220 /* TSO requires that SG is present as well. */
5221 if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosławacd11302011-01-24 15:45:15 -08005222 netdev_info(dev, "Dropping NETIF_F_TSO since no SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005223 features &= ~NETIF_F_TSO;
5224 }
5225
Michał Mirosław212b5732011-02-15 16:59:16 +00005226 /* Software GSO depends on SG. */
5227 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
5228 netdev_info(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
5229 features &= ~NETIF_F_GSO;
5230 }
5231
Michał Mirosławacd11302011-01-24 15:45:15 -08005232 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07005233 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00005234 /* maybe split UFO into V4 and V6? */
5235 if (!((features & NETIF_F_GEN_CSUM) ||
5236 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5237 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosławacd11302011-01-24 15:45:15 -08005238 netdev_info(dev,
5239 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005240 features &= ~NETIF_F_UFO;
5241 }
5242
5243 if (!(features & NETIF_F_SG)) {
Michał Mirosławacd11302011-01-24 15:45:15 -08005244 netdev_info(dev,
5245 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005246 features &= ~NETIF_F_UFO;
5247 }
5248 }
5249
5250 return features;
5251}
5252EXPORT_SYMBOL(netdev_fix_features);
5253
Michał Mirosław5455c692011-02-15 16:59:17 +00005254void netdev_update_features(struct net_device *dev)
5255{
5256 u32 features;
5257 int err = 0;
5258
5259 features = netdev_get_wanted_features(dev);
5260
5261 if (dev->netdev_ops->ndo_fix_features)
5262 features = dev->netdev_ops->ndo_fix_features(dev, features);
5263
5264 /* driver might be less strict about feature dependencies */
5265 features = netdev_fix_features(dev, features);
5266
5267 if (dev->features == features)
5268 return;
5269
5270 netdev_info(dev, "Features changed: 0x%08x -> 0x%08x\n",
5271 dev->features, features);
5272
5273 if (dev->netdev_ops->ndo_set_features)
5274 err = dev->netdev_ops->ndo_set_features(dev, features);
5275
5276 if (!err)
5277 dev->features = features;
5278 else if (err < 0)
5279 netdev_err(dev,
5280 "set_features() failed (%d); wanted 0x%08x, left 0x%08x\n",
5281 err, features, dev->features);
5282}
5283EXPORT_SYMBOL(netdev_update_features);
5284
Linus Torvalds1da177e2005-04-16 15:20:36 -07005285/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005286 * netif_stacked_transfer_operstate - transfer operstate
5287 * @rootdev: the root or lower level device to transfer state from
5288 * @dev: the device to transfer operstate to
5289 *
5290 * Transfer operational state from root to device. This is normally
5291 * called when a stacking relationship exists between the root
5292 * device and the device(a leaf device).
5293 */
5294void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5295 struct net_device *dev)
5296{
5297 if (rootdev->operstate == IF_OPER_DORMANT)
5298 netif_dormant_on(dev);
5299 else
5300 netif_dormant_off(dev);
5301
5302 if (netif_carrier_ok(rootdev)) {
5303 if (!netif_carrier_ok(dev))
5304 netif_carrier_on(dev);
5305 } else {
5306 if (netif_carrier_ok(dev))
5307 netif_carrier_off(dev);
5308 }
5309}
5310EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5311
Tom Herbertbf264142010-11-26 08:36:09 +00005312#ifdef CONFIG_RPS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005313static int netif_alloc_rx_queues(struct net_device *dev)
5314{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005315 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00005316 struct netdev_rx_queue *rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005317
Tom Herbertbd25fa72010-10-18 18:00:16 +00005318 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005319
Tom Herbertbd25fa72010-10-18 18:00:16 +00005320 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5321 if (!rx) {
5322 pr_err("netdev: Unable to allocate %u rx queues.\n", count);
5323 return -ENOMEM;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005324 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00005325 dev->_rx = rx;
5326
Tom Herbertbd25fa72010-10-18 18:00:16 +00005327 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00005328 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005329 return 0;
5330}
Tom Herbertbf264142010-11-26 08:36:09 +00005331#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005332
Changli Gaoaa942102010-12-04 02:31:41 +00005333static void netdev_init_one_queue(struct net_device *dev,
5334 struct netdev_queue *queue, void *_unused)
5335{
5336 /* Initialize queue lock */
5337 spin_lock_init(&queue->_xmit_lock);
5338 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5339 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00005340 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00005341 queue->dev = dev;
5342}
5343
Tom Herberte6484932010-10-18 18:04:39 +00005344static int netif_alloc_netdev_queues(struct net_device *dev)
5345{
5346 unsigned int count = dev->num_tx_queues;
5347 struct netdev_queue *tx;
5348
5349 BUG_ON(count < 1);
5350
5351 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5352 if (!tx) {
5353 pr_err("netdev: Unable to allocate %u tx queues.\n",
5354 count);
5355 return -ENOMEM;
5356 }
5357 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00005358
Tom Herberte6484932010-10-18 18:04:39 +00005359 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5360 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00005361
5362 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00005363}
5364
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005365/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005366 * register_netdevice - register a network device
5367 * @dev: device to register
5368 *
5369 * Take a completed network device structure and add it to the kernel
5370 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5371 * chain. 0 is returned on success. A negative errno code is returned
5372 * on a failure to set up the device, or if the name is a duplicate.
5373 *
5374 * Callers must hold the rtnl semaphore. You may want
5375 * register_netdev() instead of this.
5376 *
5377 * BUGS:
5378 * The locking appears insufficient to guarantee two parallel registers
5379 * will not get the same name.
5380 */
5381
5382int register_netdevice(struct net_device *dev)
5383{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005384 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005385 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005386
5387 BUG_ON(dev_boot_phase);
5388 ASSERT_RTNL();
5389
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005390 might_sleep();
5391
Linus Torvalds1da177e2005-04-16 15:20:36 -07005392 /* When net_device's are persistent, this will be fatal. */
5393 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005394 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005395
David S. Millerf1f28aa2008-07-15 00:08:33 -07005396 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005397 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005398
Linus Torvalds1da177e2005-04-16 15:20:36 -07005399 dev->iflink = -1;
5400
5401 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005402 if (dev->netdev_ops->ndo_init) {
5403 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005404 if (ret) {
5405 if (ret > 0)
5406 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005407 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005408 }
5409 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005410
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00005411 ret = dev_get_valid_name(dev, dev->name, 0);
Octavian Purdilad9031022009-11-18 02:36:59 +00005412 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005413 goto err_uninit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005414
Eric W. Biederman881d9662007-09-17 11:56:21 -07005415 dev->ifindex = dev_new_index(net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005416 if (dev->iflink == -1)
5417 dev->iflink = dev->ifindex;
5418
Michał Mirosław5455c692011-02-15 16:59:17 +00005419 /* Transfer changeable features to wanted_features and enable
5420 * software offloads (GSO and GRO).
5421 */
5422 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00005423 dev->features |= NETIF_F_SOFT_FEATURES;
5424 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005425
Michał Mirosław212b5732011-02-15 16:59:16 +00005426 /* Avoid warning from netdev_fix_features() for GSO without SG */
Michał Mirosław14d12322011-02-22 16:52:28 +00005427 if (!(dev->wanted_features & NETIF_F_SG)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00005428 dev->wanted_features &= ~NETIF_F_GSO;
Michał Mirosław14d12322011-02-22 16:52:28 +00005429 dev->features &= ~NETIF_F_GSO;
5430 }
Michał Mirosław212b5732011-02-15 16:59:16 +00005431
Eric Dumazetc5256c52010-09-23 00:46:11 +00005432 /* Enable GRO and NETIF_F_HIGHDMA for vlans by default,
5433 * vlan_dev_init() will do the dev->features check, so these features
5434 * are enabled only if supported by underlying device.
Brandon Philips16c3ea72010-09-15 09:24:24 +00005435 */
Eric Dumazetc5256c52010-09-23 00:46:11 +00005436 dev->vlan_features |= (NETIF_F_GRO | NETIF_F_HIGHDMA);
Brandon Philips16c3ea72010-09-15 09:24:24 +00005437
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005438 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5439 ret = notifier_to_errno(ret);
5440 if (ret)
5441 goto err_uninit;
5442
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005443 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005444 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005445 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005446 dev->reg_state = NETREG_REGISTERED;
5447
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00005448 netdev_update_features(dev);
5449
Linus Torvalds1da177e2005-04-16 15:20:36 -07005450 /*
5451 * Default initial state at registry is that the
5452 * device is present.
5453 */
5454
5455 set_bit(__LINK_STATE_PRESENT, &dev->state);
5456
Linus Torvalds1da177e2005-04-16 15:20:36 -07005457 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005458 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005459 list_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005460
5461 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005462 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005463 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005464 if (ret) {
5465 rollback_registered(dev);
5466 dev->reg_state = NETREG_UNREGISTERED;
5467 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005468 /*
5469 * Prevent userspace races by waiting until the network
5470 * device is fully setup before sending notifications.
5471 */
Patrick McHardya2835762010-02-26 06:34:51 +00005472 if (!dev->rtnl_link_ops ||
5473 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5474 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005475
5476out:
5477 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005478
5479err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005480 if (dev->netdev_ops->ndo_uninit)
5481 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005482 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005483}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005484EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005485
5486/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005487 * init_dummy_netdev - init a dummy network device for NAPI
5488 * @dev: device to init
5489 *
5490 * This takes a network device structure and initialize the minimum
5491 * amount of fields so it can be used to schedule NAPI polls without
5492 * registering a full blown interface. This is to be used by drivers
5493 * that need to tie several hardware interfaces to a single NAPI
5494 * poll scheduler due to HW limitations.
5495 */
5496int init_dummy_netdev(struct net_device *dev)
5497{
5498 /* Clear everything. Note we don't initialize spinlocks
5499 * are they aren't supposed to be taken by any of the
5500 * NAPI code and this dummy netdev is supposed to be
5501 * only ever used for NAPI polls
5502 */
5503 memset(dev, 0, sizeof(struct net_device));
5504
5505 /* make sure we BUG if trying to hit standard
5506 * register/unregister code path
5507 */
5508 dev->reg_state = NETREG_DUMMY;
5509
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005510 /* NAPI wants this */
5511 INIT_LIST_HEAD(&dev->napi_list);
5512
5513 /* a dummy interface is started by default */
5514 set_bit(__LINK_STATE_PRESENT, &dev->state);
5515 set_bit(__LINK_STATE_START, &dev->state);
5516
Eric Dumazet29b44332010-10-11 10:22:12 +00005517 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5518 * because users of this 'device' dont need to change
5519 * its refcount.
5520 */
5521
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005522 return 0;
5523}
5524EXPORT_SYMBOL_GPL(init_dummy_netdev);
5525
5526
5527/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005528 * register_netdev - register a network device
5529 * @dev: device to register
5530 *
5531 * Take a completed network device structure and add it to the kernel
5532 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5533 * chain. 0 is returned on success. A negative errno code is returned
5534 * on a failure to set up the device, or if the name is a duplicate.
5535 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005536 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005537 * and expands the device name if you passed a format string to
5538 * alloc_netdev.
5539 */
5540int register_netdev(struct net_device *dev)
5541{
5542 int err;
5543
5544 rtnl_lock();
5545
5546 /*
5547 * If the name is a format string the caller wants us to do a
5548 * name allocation.
5549 */
5550 if (strchr(dev->name, '%')) {
5551 err = dev_alloc_name(dev, dev->name);
5552 if (err < 0)
5553 goto out;
5554 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005555
Linus Torvalds1da177e2005-04-16 15:20:36 -07005556 err = register_netdevice(dev);
5557out:
5558 rtnl_unlock();
5559 return err;
5560}
5561EXPORT_SYMBOL(register_netdev);
5562
Eric Dumazet29b44332010-10-11 10:22:12 +00005563int netdev_refcnt_read(const struct net_device *dev)
5564{
5565 int i, refcnt = 0;
5566
5567 for_each_possible_cpu(i)
5568 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5569 return refcnt;
5570}
5571EXPORT_SYMBOL(netdev_refcnt_read);
5572
Linus Torvalds1da177e2005-04-16 15:20:36 -07005573/*
5574 * netdev_wait_allrefs - wait until all references are gone.
5575 *
5576 * This is called when unregistering network devices.
5577 *
5578 * Any protocol or device that holds a reference should register
5579 * for netdevice notification, and cleanup and put back the
5580 * reference if they receive an UNREGISTER event.
5581 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005582 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005583 */
5584static void netdev_wait_allrefs(struct net_device *dev)
5585{
5586 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00005587 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005588
Eric Dumazete014deb2009-11-17 05:59:21 +00005589 linkwatch_forget_dev(dev);
5590
Linus Torvalds1da177e2005-04-16 15:20:36 -07005591 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00005592 refcnt = netdev_refcnt_read(dev);
5593
5594 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005595 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005596 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005597
5598 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005599 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005600 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
Octavian Purdila395264d2009-11-16 13:49:35 +00005601 * should have already handle it the first time */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005602
5603 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5604 &dev->state)) {
5605 /* We must not have linkwatch events
5606 * pending on unregister. If this
5607 * happens, we simply run the queue
5608 * unscheduled, resulting in a noop
5609 * for this device.
5610 */
5611 linkwatch_run_queue();
5612 }
5613
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005614 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005615
5616 rebroadcast_time = jiffies;
5617 }
5618
5619 msleep(250);
5620
Eric Dumazet29b44332010-10-11 10:22:12 +00005621 refcnt = netdev_refcnt_read(dev);
5622
Linus Torvalds1da177e2005-04-16 15:20:36 -07005623 if (time_after(jiffies, warning_time + 10 * HZ)) {
5624 printk(KERN_EMERG "unregister_netdevice: "
5625 "waiting for %s to become free. Usage "
5626 "count = %d\n",
Eric Dumazet29b44332010-10-11 10:22:12 +00005627 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005628 warning_time = jiffies;
5629 }
5630 }
5631}
5632
5633/* The sequence is:
5634 *
5635 * rtnl_lock();
5636 * ...
5637 * register_netdevice(x1);
5638 * register_netdevice(x2);
5639 * ...
5640 * unregister_netdevice(y1);
5641 * unregister_netdevice(y2);
5642 * ...
5643 * rtnl_unlock();
5644 * free_netdev(y1);
5645 * free_netdev(y2);
5646 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005647 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005648 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005649 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005650 * without deadlocking with linkwatch via keventd.
5651 * 2) Since we run with the RTNL semaphore not held, we can sleep
5652 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005653 *
5654 * We must not return until all unregister events added during
5655 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005656 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005657void netdev_run_todo(void)
5658{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005659 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005660
Linus Torvalds1da177e2005-04-16 15:20:36 -07005661 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005662 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07005663
5664 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005665
Linus Torvalds1da177e2005-04-16 15:20:36 -07005666 while (!list_empty(&list)) {
5667 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00005668 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005669 list_del(&dev->todo_list);
5670
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005671 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005672 printk(KERN_ERR "network todo '%s' but state %d\n",
5673 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005674 dump_stack();
5675 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005676 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005677
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005678 dev->reg_state = NETREG_UNREGISTERED;
5679
Changli Gao152102c2010-03-30 20:16:22 +00005680 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005681
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005682 netdev_wait_allrefs(dev);
5683
5684 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00005685 BUG_ON(netdev_refcnt_read(dev));
Eric Dumazet95ae6b22010-09-15 04:04:31 +00005686 WARN_ON(rcu_dereference_raw(dev->ip_ptr));
Eric Dumazet198caec2010-10-24 21:32:05 +00005687 WARN_ON(rcu_dereference_raw(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07005688 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005689
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005690 if (dev->destructor)
5691 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07005692
5693 /* Free network device */
5694 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005695 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005696}
5697
Ben Hutchings3cfde792010-07-09 09:11:52 +00005698/* Convert net_device_stats to rtnl_link_stats64. They have the same
5699 * fields in the same order, with only the type differing.
5700 */
5701static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5702 const struct net_device_stats *netdev_stats)
5703{
5704#if BITS_PER_LONG == 64
5705 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5706 memcpy(stats64, netdev_stats, sizeof(*stats64));
5707#else
5708 size_t i, n = sizeof(*stats64) / sizeof(u64);
5709 const unsigned long *src = (const unsigned long *)netdev_stats;
5710 u64 *dst = (u64 *)stats64;
5711
5712 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5713 sizeof(*stats64) / sizeof(u64));
5714 for (i = 0; i < n; i++)
5715 dst[i] = src[i];
5716#endif
5717}
5718
Eric Dumazetd83345a2009-11-16 03:36:51 +00005719/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005720 * dev_get_stats - get network device statistics
5721 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07005722 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005723 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00005724 * Get network statistics from device. Return @storage.
5725 * The device driver may provide its own method by setting
5726 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5727 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005728 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00005729struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5730 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00005731{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005732 const struct net_device_ops *ops = dev->netdev_ops;
5733
Eric Dumazet28172732010-07-07 14:58:56 -07005734 if (ops->ndo_get_stats64) {
5735 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005736 ops->ndo_get_stats64(dev, storage);
5737 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00005738 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005739 } else {
5740 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07005741 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005742 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07005743 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07005744}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005745EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07005746
Eric Dumazet24824a02010-10-02 06:11:55 +00005747struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07005748{
Eric Dumazet24824a02010-10-02 06:11:55 +00005749 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07005750
Eric Dumazet24824a02010-10-02 06:11:55 +00005751#ifdef CONFIG_NET_CLS_ACT
5752 if (queue)
5753 return queue;
5754 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5755 if (!queue)
5756 return NULL;
5757 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet24824a02010-10-02 06:11:55 +00005758 queue->qdisc = &noop_qdisc;
5759 queue->qdisc_sleeping = &noop_qdisc;
5760 rcu_assign_pointer(dev->ingress_queue, queue);
5761#endif
5762 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07005763}
5764
Linus Torvalds1da177e2005-04-16 15:20:36 -07005765/**
Tom Herbert36909ea2011-01-09 19:36:31 +00005766 * alloc_netdev_mqs - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005767 * @sizeof_priv: size of private data to allocate space for
5768 * @name: device name format string
5769 * @setup: callback to initialize device
Tom Herbert36909ea2011-01-09 19:36:31 +00005770 * @txqs: the number of TX subqueues to allocate
5771 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07005772 *
5773 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005774 * and performs basic initialization. Also allocates subquue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00005775 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005776 */
Tom Herbert36909ea2011-01-09 19:36:31 +00005777struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5778 void (*setup)(struct net_device *),
5779 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005780{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005781 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07005782 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005783 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005784
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005785 BUG_ON(strlen(name) >= sizeof(dev->name));
5786
Tom Herbert36909ea2011-01-09 19:36:31 +00005787 if (txqs < 1) {
Tom Herbert55513fb2010-10-18 17:55:58 +00005788 pr_err("alloc_netdev: Unable to allocate device "
5789 "with zero queues.\n");
5790 return NULL;
5791 }
5792
Tom Herbert36909ea2011-01-09 19:36:31 +00005793#ifdef CONFIG_RPS
5794 if (rxqs < 1) {
5795 pr_err("alloc_netdev: Unable to allocate device "
5796 "with zero RX queues.\n");
5797 return NULL;
5798 }
5799#endif
5800
David S. Millerfd2ea0a2008-07-17 01:56:23 -07005801 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005802 if (sizeof_priv) {
5803 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005804 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005805 alloc_size += sizeof_priv;
5806 }
5807 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005808 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005809
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07005810 p = kzalloc(alloc_size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005811 if (!p) {
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005812 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005813 return NULL;
5814 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005815
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005816 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005817 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005818
Eric Dumazet29b44332010-10-11 10:22:12 +00005819 dev->pcpu_refcnt = alloc_percpu(int);
5820 if (!dev->pcpu_refcnt)
Tom Herberte6484932010-10-18 18:04:39 +00005821 goto free_p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005822
Linus Torvalds1da177e2005-04-16 15:20:36 -07005823 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00005824 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005825
Jiri Pirko22bedad32010-04-01 21:22:57 +00005826 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005827 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00005828
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09005829 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005830
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07005831 dev->gso_max_size = GSO_MAX_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005832
Peter P Waskiewicz Jr15682bc2010-02-10 20:03:05 -08005833 INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list);
5834 dev->ethtool_ntuple_list.count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005835 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005836 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00005837 INIT_LIST_HEAD(&dev->link_watch_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07005838 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005839 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08005840
5841 dev->num_tx_queues = txqs;
5842 dev->real_num_tx_queues = txqs;
5843 if (netif_alloc_netdev_queues(dev))
5844 goto free_all;
5845
5846#ifdef CONFIG_RPS
5847 dev->num_rx_queues = rxqs;
5848 dev->real_num_rx_queues = rxqs;
5849 if (netif_alloc_rx_queues(dev))
5850 goto free_all;
5851#endif
5852
Linus Torvalds1da177e2005-04-16 15:20:36 -07005853 strcpy(dev->name, name);
Vlad Dogarucbda10f2011-01-13 23:38:30 +00005854 dev->group = INIT_NETDEV_GROUP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005855 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005856
David S. Miller8d3bdbd2011-02-08 15:02:50 -08005857free_all:
5858 free_netdev(dev);
5859 return NULL;
5860
Eric Dumazet29b44332010-10-11 10:22:12 +00005861free_pcpu:
5862 free_percpu(dev->pcpu_refcnt);
Tom Herberted9af2e2010-11-09 10:47:30 +00005863 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00005864#ifdef CONFIG_RPS
5865 kfree(dev->_rx);
5866#endif
5867
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005868free_p:
5869 kfree(p);
5870 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005871}
Tom Herbert36909ea2011-01-09 19:36:31 +00005872EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005873
5874/**
5875 * free_netdev - free network device
5876 * @dev: device
5877 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005878 * This function does the last stage of destroying an allocated device
5879 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005880 * If this is the last reference then it will be freed.
5881 */
5882void free_netdev(struct net_device *dev)
5883{
Herbert Xud565b0a2008-12-15 23:38:52 -08005884 struct napi_struct *p, *n;
5885
Denis V. Lunevf3005d72008-04-16 02:02:18 -07005886 release_net(dev_net(dev));
5887
David S. Millere8a04642008-07-17 00:34:19 -07005888 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00005889#ifdef CONFIG_RPS
5890 kfree(dev->_rx);
5891#endif
David S. Millere8a04642008-07-17 00:34:19 -07005892
Eric Dumazet24824a02010-10-02 06:11:55 +00005893 kfree(rcu_dereference_raw(dev->ingress_queue));
5894
Jiri Pirkof001fde2009-05-05 02:48:28 +00005895 /* Flush device addresses */
5896 dev_addr_flush(dev);
5897
Peter P Waskiewicz Jr15682bc2010-02-10 20:03:05 -08005898 /* Clear ethtool n-tuple list */
5899 ethtool_ntuple_flush(dev);
5900
Herbert Xud565b0a2008-12-15 23:38:52 -08005901 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5902 netif_napi_del(p);
5903
Eric Dumazet29b44332010-10-11 10:22:12 +00005904 free_percpu(dev->pcpu_refcnt);
5905 dev->pcpu_refcnt = NULL;
5906
Stephen Hemminger3041a062006-05-26 13:25:24 -07005907 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005908 if (dev->reg_state == NETREG_UNINITIALIZED) {
5909 kfree((char *)dev - dev->padded);
5910 return;
5911 }
5912
5913 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5914 dev->reg_state = NETREG_RELEASED;
5915
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07005916 /* will free via device release */
5917 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005918}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005919EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005920
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005921/**
5922 * synchronize_net - Synchronize with packet receive processing
5923 *
5924 * Wait for packets currently being received to be done.
5925 * Does not block later packets from starting.
5926 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005927void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005928{
5929 might_sleep();
Paul E. McKenneyfbd568a3e2005-05-01 08:59:04 -07005930 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005931}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005932EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005933
5934/**
Eric Dumazet44a08732009-10-27 07:03:04 +00005935 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07005936 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00005937 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08005938 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07005939 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08005940 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00005941 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005942 *
5943 * Callers must hold the rtnl semaphore. You may want
5944 * unregister_netdev() instead of this.
5945 */
5946
Eric Dumazet44a08732009-10-27 07:03:04 +00005947void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005948{
Herbert Xua6620712007-12-12 19:21:56 -08005949 ASSERT_RTNL();
5950
Eric Dumazet44a08732009-10-27 07:03:04 +00005951 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005952 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00005953 } else {
5954 rollback_registered(dev);
5955 /* Finish processing unregister after unlock */
5956 net_set_todo(dev);
5957 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005958}
Eric Dumazet44a08732009-10-27 07:03:04 +00005959EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005960
5961/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005962 * unregister_netdevice_many - unregister many devices
5963 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005964 */
5965void unregister_netdevice_many(struct list_head *head)
5966{
5967 struct net_device *dev;
5968
5969 if (!list_empty(head)) {
5970 rollback_registered_many(head);
5971 list_for_each_entry(dev, head, unreg_list)
5972 net_set_todo(dev);
5973 }
5974}
Eric Dumazet63c80992009-10-27 07:06:49 +00005975EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005976
5977/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005978 * unregister_netdev - remove device from the kernel
5979 * @dev: device
5980 *
5981 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08005982 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005983 *
5984 * This is just a wrapper for unregister_netdevice that takes
5985 * the rtnl semaphore. In general you want to use this and not
5986 * unregister_netdevice.
5987 */
5988void unregister_netdev(struct net_device *dev)
5989{
5990 rtnl_lock();
5991 unregister_netdevice(dev);
5992 rtnl_unlock();
5993}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005994EXPORT_SYMBOL(unregister_netdev);
5995
Eric W. Biedermance286d32007-09-12 13:53:49 +02005996/**
5997 * dev_change_net_namespace - move device to different nethost namespace
5998 * @dev: device
5999 * @net: network namespace
6000 * @pat: If not NULL name pattern to try if the current device name
6001 * is already taken in the destination network namespace.
6002 *
6003 * This function shuts down a device interface and moves it
6004 * to a new network namespace. On success 0 is returned, on
6005 * a failure a netagive errno code is returned.
6006 *
6007 * Callers must hold the rtnl semaphore.
6008 */
6009
6010int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
6011{
Eric W. Biedermance286d32007-09-12 13:53:49 +02006012 int err;
6013
6014 ASSERT_RTNL();
6015
6016 /* Don't allow namespace local devices to be moved. */
6017 err = -EINVAL;
6018 if (dev->features & NETIF_F_NETNS_LOCAL)
6019 goto out;
6020
6021 /* Ensure the device has been registrered */
6022 err = -EINVAL;
6023 if (dev->reg_state != NETREG_REGISTERED)
6024 goto out;
6025
6026 /* Get out if there is nothing todo */
6027 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09006028 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02006029 goto out;
6030
6031 /* Pick the destination device name, and ensure
6032 * we can use it in the destination network namespace.
6033 */
6034 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00006035 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006036 /* We get here if we can't use the current device name */
6037 if (!pat)
6038 goto out;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00006039 if (dev_get_valid_name(dev, pat, 1))
Eric W. Biedermance286d32007-09-12 13:53:49 +02006040 goto out;
6041 }
6042
6043 /*
6044 * And now a mini version of register_netdevice unregister_netdevice.
6045 */
6046
6047 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07006048 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006049
6050 /* And unlink it from device chain */
6051 err = -ENODEV;
6052 unlist_netdevice(dev);
6053
6054 synchronize_net();
6055
6056 /* Shutdown queueing discipline. */
6057 dev_shutdown(dev);
6058
6059 /* Notify protocols, that we are about to destroy
6060 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00006061
6062 Note that dev->reg_state stays at NETREG_REGISTERED.
6063 This is wanted because this way 8021q and macvlan know
6064 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02006065 */
6066 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00006067 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006068
6069 /*
6070 * Flush the unicast and multicast chains
6071 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006072 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00006073 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006074
6075 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006076 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006077
Eric W. Biedermance286d32007-09-12 13:53:49 +02006078 /* If there is an ifindex conflict assign a new one */
6079 if (__dev_get_by_index(net, dev->ifindex)) {
6080 int iflink = (dev->iflink == dev->ifindex);
6081 dev->ifindex = dev_new_index(net);
6082 if (iflink)
6083 dev->iflink = dev->ifindex;
6084 }
6085
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006086 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07006087 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006088 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006089
6090 /* Add the device back in the hashes */
6091 list_netdevice(dev);
6092
6093 /* Notify protocols, that a new device appeared. */
6094 call_netdevice_notifiers(NETDEV_REGISTER, dev);
6095
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006096 /*
6097 * Prevent userspace races by waiting until the network
6098 * device is fully setup before sending notifications.
6099 */
6100 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
6101
Eric W. Biedermance286d32007-09-12 13:53:49 +02006102 synchronize_net();
6103 err = 0;
6104out:
6105 return err;
6106}
Johannes Berg463d0182009-07-14 00:33:35 +02006107EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006108
Linus Torvalds1da177e2005-04-16 15:20:36 -07006109static int dev_cpu_callback(struct notifier_block *nfb,
6110 unsigned long action,
6111 void *ocpu)
6112{
6113 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006114 struct sk_buff *skb;
6115 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6116 struct softnet_data *sd, *oldsd;
6117
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07006118 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006119 return NOTIFY_OK;
6120
6121 local_irq_disable();
6122 cpu = smp_processor_id();
6123 sd = &per_cpu(softnet_data, cpu);
6124 oldsd = &per_cpu(softnet_data, oldcpu);
6125
6126 /* Find end of our completion_queue. */
6127 list_skb = &sd->completion_queue;
6128 while (*list_skb)
6129 list_skb = &(*list_skb)->next;
6130 /* Append completion queue from offline CPU. */
6131 *list_skb = oldsd->completion_queue;
6132 oldsd->completion_queue = NULL;
6133
Linus Torvalds1da177e2005-04-16 15:20:36 -07006134 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00006135 if (oldsd->output_queue) {
6136 *sd->output_queue_tailp = oldsd->output_queue;
6137 sd->output_queue_tailp = oldsd->output_queue_tailp;
6138 oldsd->output_queue = NULL;
6139 oldsd->output_queue_tailp = &oldsd->output_queue;
6140 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006141
6142 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6143 local_irq_enable();
6144
6145 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00006146 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6147 netif_rx(skb);
6148 input_queue_head_incr(oldsd);
6149 }
Tom Herbertfec5e652010-04-16 16:01:27 -07006150 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006151 netif_rx(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00006152 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07006153 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006154
6155 return NOTIFY_OK;
6156}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006157
6158
Herbert Xu7f353bf2007-08-10 15:47:58 -07006159/**
Herbert Xub63365a2008-10-23 01:11:29 -07006160 * netdev_increment_features - increment feature set by one
6161 * @all: current feature set
6162 * @one: new feature set
6163 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07006164 *
6165 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07006166 * @one to the master device with current feature set @all. Will not
6167 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07006168 */
Michał Mirosław04ed3e72011-01-24 15:32:47 -08006169u32 netdev_increment_features(u32 all, u32 one, u32 mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07006170{
Herbert Xub63365a2008-10-23 01:11:29 -07006171 /* If device needs checksumming, downgrade to it. */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006172 if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))
Herbert Xub63365a2008-10-23 01:11:29 -07006173 all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);
6174 else if (mask & NETIF_F_ALL_CSUM) {
6175 /* If one device supports v4/v6 checksumming, set for all. */
6176 if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&
6177 !(all & NETIF_F_GEN_CSUM)) {
6178 all &= ~NETIF_F_ALL_CSUM;
6179 all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
6180 }
Herbert Xu7f353bf2007-08-10 15:47:58 -07006181
Herbert Xub63365a2008-10-23 01:11:29 -07006182 /* If one device supports hw checksumming, set for all. */
6183 if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {
6184 all &= ~NETIF_F_ALL_CSUM;
6185 all |= NETIF_F_HW_CSUM;
6186 }
6187 }
Herbert Xu7f353bf2007-08-10 15:47:58 -07006188
Herbert Xub63365a2008-10-23 01:11:29 -07006189 one |= NETIF_F_ALL_CSUM;
Herbert Xu7f353bf2007-08-10 15:47:58 -07006190
Herbert Xub63365a2008-10-23 01:11:29 -07006191 one |= all & NETIF_F_ONE_FOR_ALL;
Sridhar Samudralad9f59502009-10-07 12:24:25 +00006192 all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO;
Herbert Xub63365a2008-10-23 01:11:29 -07006193 all |= one & mask & NETIF_F_ONE_FOR_ALL;
Herbert Xu7f353bf2007-08-10 15:47:58 -07006194
6195 return all;
6196}
Herbert Xub63365a2008-10-23 01:11:29 -07006197EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006198
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006199static struct hlist_head *netdev_create_hash(void)
6200{
6201 int i;
6202 struct hlist_head *hash;
6203
6204 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6205 if (hash != NULL)
6206 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6207 INIT_HLIST_HEAD(&hash[i]);
6208
6209 return hash;
6210}
6211
Eric W. Biederman881d9662007-09-17 11:56:21 -07006212/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07006213static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006214{
Eric W. Biederman881d9662007-09-17 11:56:21 -07006215 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07006216
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006217 net->dev_name_head = netdev_create_hash();
6218 if (net->dev_name_head == NULL)
6219 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006220
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006221 net->dev_index_head = netdev_create_hash();
6222 if (net->dev_index_head == NULL)
6223 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006224
6225 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006226
6227err_idx:
6228 kfree(net->dev_name_head);
6229err_name:
6230 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006231}
6232
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006233/**
6234 * netdev_drivername - network driver for the device
6235 * @dev: network device
6236 * @buffer: buffer for resulting name
6237 * @len: size of buffer
6238 *
6239 * Determine network driver for device.
6240 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006241char *netdev_drivername(const struct net_device *dev, char *buffer, int len)
Arjan van de Ven6579e572008-07-21 13:31:48 -07006242{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006243 const struct device_driver *driver;
6244 const struct device *parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006245
6246 if (len <= 0 || !buffer)
6247 return buffer;
6248 buffer[0] = 0;
6249
6250 parent = dev->dev.parent;
6251
6252 if (!parent)
6253 return buffer;
6254
6255 driver = parent->driver;
6256 if (driver && driver->name)
6257 strlcpy(buffer, driver->name, len);
6258 return buffer;
6259}
6260
Joe Perches256df2f2010-06-27 01:02:35 +00006261static int __netdev_printk(const char *level, const struct net_device *dev,
6262 struct va_format *vaf)
6263{
6264 int r;
6265
6266 if (dev && dev->dev.parent)
6267 r = dev_printk(level, dev->dev.parent, "%s: %pV",
6268 netdev_name(dev), vaf);
6269 else if (dev)
6270 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6271 else
6272 r = printk("%s(NULL net_device): %pV", level, vaf);
6273
6274 return r;
6275}
6276
6277int netdev_printk(const char *level, const struct net_device *dev,
6278 const char *format, ...)
6279{
6280 struct va_format vaf;
6281 va_list args;
6282 int r;
6283
6284 va_start(args, format);
6285
6286 vaf.fmt = format;
6287 vaf.va = &args;
6288
6289 r = __netdev_printk(level, dev, &vaf);
6290 va_end(args);
6291
6292 return r;
6293}
6294EXPORT_SYMBOL(netdev_printk);
6295
6296#define define_netdev_printk_level(func, level) \
6297int func(const struct net_device *dev, const char *fmt, ...) \
6298{ \
6299 int r; \
6300 struct va_format vaf; \
6301 va_list args; \
6302 \
6303 va_start(args, fmt); \
6304 \
6305 vaf.fmt = fmt; \
6306 vaf.va = &args; \
6307 \
6308 r = __netdev_printk(level, dev, &vaf); \
6309 va_end(args); \
6310 \
6311 return r; \
6312} \
6313EXPORT_SYMBOL(func);
6314
6315define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6316define_netdev_printk_level(netdev_alert, KERN_ALERT);
6317define_netdev_printk_level(netdev_crit, KERN_CRIT);
6318define_netdev_printk_level(netdev_err, KERN_ERR);
6319define_netdev_printk_level(netdev_warn, KERN_WARNING);
6320define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6321define_netdev_printk_level(netdev_info, KERN_INFO);
6322
Pavel Emelyanov46650792007-10-08 20:38:39 -07006323static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006324{
6325 kfree(net->dev_name_head);
6326 kfree(net->dev_index_head);
6327}
6328
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006329static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006330 .init = netdev_init,
6331 .exit = netdev_exit,
6332};
6333
Pavel Emelyanov46650792007-10-08 20:38:39 -07006334static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006335{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006336 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006337 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006338 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006339 * initial network namespace
6340 */
6341 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006342 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006343 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006344 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006345
6346 /* Ignore unmoveable devices (i.e. loopback) */
6347 if (dev->features & NETIF_F_NETNS_LOCAL)
6348 continue;
6349
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006350 /* Leave virtual devices for the generic cleanup */
6351 if (dev->rtnl_link_ops)
6352 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006353
Eric W. Biedermance286d32007-09-12 13:53:49 +02006354 /* Push remaing network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006355 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6356 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006357 if (err) {
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006358 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
Eric W. Biedermance286d32007-09-12 13:53:49 +02006359 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006360 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006361 }
6362 }
6363 rtnl_unlock();
6364}
6365
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006366static void __net_exit default_device_exit_batch(struct list_head *net_list)
6367{
6368 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04006369 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006370 * Do this across as many network namespaces as possible to
6371 * improve batching efficiency.
6372 */
6373 struct net_device *dev;
6374 struct net *net;
6375 LIST_HEAD(dev_kill_list);
6376
6377 rtnl_lock();
6378 list_for_each_entry(net, net_list, exit_list) {
6379 for_each_netdev_reverse(net, dev) {
6380 if (dev->rtnl_link_ops)
6381 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6382 else
6383 unregister_netdevice_queue(dev, &dev_kill_list);
6384 }
6385 }
6386 unregister_netdevice_many(&dev_kill_list);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006387 list_del(&dev_kill_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006388 rtnl_unlock();
6389}
6390
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006391static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006392 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006393 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006394};
6395
Linus Torvalds1da177e2005-04-16 15:20:36 -07006396/*
6397 * Initialize the DEV module. At boot time this walks the device list and
6398 * unhooks any devices that fail to initialise (normally hardware not
6399 * present) and leaves us with a valid list of present and active devices.
6400 *
6401 */
6402
6403/*
6404 * This is called single threaded during boot, so no need
6405 * to take the rtnl semaphore.
6406 */
6407static int __init net_dev_init(void)
6408{
6409 int i, rc = -ENOMEM;
6410
6411 BUG_ON(!dev_boot_phase);
6412
Linus Torvalds1da177e2005-04-16 15:20:36 -07006413 if (dev_proc_init())
6414 goto out;
6415
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006416 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006417 goto out;
6418
6419 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006420 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006421 INIT_LIST_HEAD(&ptype_base[i]);
6422
Eric W. Biederman881d9662007-09-17 11:56:21 -07006423 if (register_pernet_subsys(&netdev_net_ops))
6424 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006425
6426 /*
6427 * Initialise the packet receive queues.
6428 */
6429
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006430 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006431 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006432
Changli Gaodee42872010-05-02 05:42:16 +00006433 memset(sd, 0, sizeof(*sd));
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006434 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006435 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006436 sd->completion_queue = NULL;
6437 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00006438 sd->output_queue = NULL;
6439 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00006440#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006441 sd->csd.func = rps_trigger_softirq;
6442 sd->csd.info = sd;
6443 sd->csd.flags = 0;
6444 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07006445#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006446
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006447 sd->backlog.poll = process_backlog;
6448 sd->backlog.weight = weight_p;
6449 sd->backlog.gro_list = NULL;
6450 sd->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006451 }
6452
Linus Torvalds1da177e2005-04-16 15:20:36 -07006453 dev_boot_phase = 0;
6454
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006455 /* The loopback device is special if any other network devices
6456 * is present in a network namespace the loopback device must
6457 * be present. Since we now dynamically allocate and free the
6458 * loopback device ensure this invariant is maintained by
6459 * keeping the loopback device as the first device on the
6460 * list of network devices. Ensuring the loopback devices
6461 * is the first device that appears and the last network device
6462 * that disappears.
6463 */
6464 if (register_pernet_device(&loopback_net_ops))
6465 goto out;
6466
6467 if (register_pernet_device(&default_device_ops))
6468 goto out;
6469
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006470 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6471 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006472
6473 hotcpu_notifier(dev_cpu_callback, 0);
6474 dst_init();
6475 dev_mcast_init();
6476 rc = 0;
6477out:
6478 return rc;
6479}
6480
6481subsys_initcall(net_dev_init);
6482
Krishna Kumare88721f2009-02-18 17:55:02 -08006483static int __init initialize_hashrnd(void)
6484{
Tom Herbert0a9627f2010-03-16 08:03:29 +00006485 get_random_bytes(&hashrnd, sizeof(hashrnd));
Krishna Kumare88721f2009-02-18 17:55:02 -08006486 return 0;
6487}
6488
6489late_initcall_sync(initialize_hashrnd);
6490