blob: cca02ae7a844ffa5d9bb3897dd1e2998a41ca653 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
99#include <linux/rtnetlink.h>
100#include <linux/proc_fs.h>
101#include <linux/seq_file.h>
102#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <net/dst.h>
104#include <net/pkt_sched.h>
105#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000106#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <linux/highmem.h>
108#include <linux/init.h>
109#include <linux/kmod.h>
110#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <linux/netpoll.h>
112#include <linux/rcupdate.h>
113#include <linux/delay.h>
Johannes Berg295f4a12007-04-26 20:43:56 -0700114#include <net/wext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500117#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700118#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700119#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700120#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700121#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700122#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700123#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700124#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700125#include <linux/ipv6.h>
126#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700127#include <linux/jhash.h>
128#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700129#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900130#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900131#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000132#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700133#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000134#include <linux/cpu_rmap.h>
Richard Cochran4dc360c2011-10-19 17:00:35 -0400135#include <linux/net_tstamp.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100136#include <linux/static_key.h>
Eric Dumazet4504b862011-11-28 05:23:23 +0000137#include <net/flow_keys.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700139#include "net-sysfs.h"
140
Herbert Xud565b0a2008-12-15 23:38:52 -0800141/* Instead of increasing this, you should create a hash table. */
142#define MAX_GRO_SKBS 8
143
Herbert Xu5d38a072009-01-04 16:13:40 -0800144/* This should be increased if a protocol with a bigger head is added. */
145#define GRO_MAX_HEAD (MAX_HEADER + 128)
146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147/*
148 * The list of packet types we will receive (as opposed to discard)
149 * and the routines to invoke.
150 *
151 * Why 16. Because with 16 the only overlap we get on a hash of the
152 * low nibble of the protocol value is RARP/SNAP/X.25.
153 *
154 * NOTE: That is no longer true with the addition of VLAN tags. Not
155 * sure which should go first, but I bet it won't make much
156 * difference if we are running VLANs. The good news is that
157 * this protocol won't be in the list unless compiled in, so
Stephen Hemminger3041a062006-05-26 13:25:24 -0700158 * the average user (w/out VLANs) will not be adversely affected.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 * --BLG
160 *
161 * 0800 IP
162 * 8100 802.1Q VLAN
163 * 0001 802.3
164 * 0002 AX.25
165 * 0004 802.2
166 * 8035 RARP
167 * 0005 SNAP
168 * 0805 X.25
169 * 0806 ARP
170 * 8137 IPX
171 * 0009 Localtalk
172 * 86DD IPv6
173 */
174
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800175#define PTYPE_HASH_SIZE (16)
176#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
177
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178static DEFINE_SPINLOCK(ptype_lock);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800179static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -0700180static struct list_head ptype_all __read_mostly; /* Taps */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700183 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 * semaphore.
185 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800186 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 *
188 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700189 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 * actual updates. This allows pure readers to access the list even
191 * while a writer is preparing to update it.
192 *
193 * To put it another way, dev_base_lock is held for writing only to
194 * protect against pure readers; the rtnl semaphore provides the
195 * protection against other writers.
196 *
197 * See, for example usages, register_netdevice() and
198 * unregister_netdevice(), which must be called with the rtnl
199 * semaphore held.
200 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202EXPORT_SYMBOL(dev_base_lock);
203
Thomas Graf4e985ad2011-06-21 03:11:20 +0000204static inline void dev_base_seq_inc(struct net *net)
205{
206 while (++net->dev_base_seq == 0);
207}
208
Eric W. Biederman881d9662007-09-17 11:56:21 -0700209static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210{
Eric Dumazet95c96172012-04-15 05:58:06 +0000211 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
212
stephen hemminger08e98972009-11-10 07:20:34 +0000213 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214}
215
Eric W. Biederman881d9662007-09-17 11:56:21 -0700216static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700218 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219}
220
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000221static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000222{
223#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000224 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000225#endif
226}
227
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000228static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000229{
230#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000231 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000232#endif
233}
234
Eric W. Biedermance286d32007-09-12 13:53:49 +0200235/* Device list insertion */
236static int list_netdevice(struct net_device *dev)
237{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900238 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200239
240 ASSERT_RTNL();
241
242 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800243 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000244 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000245 hlist_add_head_rcu(&dev->index_hlist,
246 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200247 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000248
249 dev_base_seq_inc(net);
250
Eric W. Biedermance286d32007-09-12 13:53:49 +0200251 return 0;
252}
253
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000254/* Device list removal
255 * caller must respect a RCU grace period before freeing/reusing dev
256 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200257static void unlist_netdevice(struct net_device *dev)
258{
259 ASSERT_RTNL();
260
261 /* Unlink dev from the device chain */
262 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800263 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000264 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000265 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200266 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000267
268 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200269}
270
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271/*
272 * Our notifier list
273 */
274
Alan Sternf07d5b92006-05-09 15:23:03 -0700275static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276
277/*
278 * Device drivers call our routines to queue packets here. We empty the
279 * queue in the local softnet handler.
280 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700281
Eric Dumazet9958da02010-04-17 04:17:02 +0000282DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700283EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284
David S. Millercf508b12008-07-22 14:16:42 -0700285#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700286/*
David S. Millerc773e842008-07-08 23:13:53 -0700287 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700288 * according to dev->type
289 */
290static const unsigned short netdev_lock_type[] =
291 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
292 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
293 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
294 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
295 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
296 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
297 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
298 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
299 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
300 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
301 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
302 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400303 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
304 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
305 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700306
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700307static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700308 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
309 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
310 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
311 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
312 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
313 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
314 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
315 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
316 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
317 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
318 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
319 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400320 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
321 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
322 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700323
324static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700325static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700326
327static inline unsigned short netdev_lock_pos(unsigned short dev_type)
328{
329 int i;
330
331 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
332 if (netdev_lock_type[i] == dev_type)
333 return i;
334 /* the last key is used by default */
335 return ARRAY_SIZE(netdev_lock_type) - 1;
336}
337
David S. Millercf508b12008-07-22 14:16:42 -0700338static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
339 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700340{
341 int i;
342
343 i = netdev_lock_pos(dev_type);
344 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
345 netdev_lock_name[i]);
346}
David S. Millercf508b12008-07-22 14:16:42 -0700347
348static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
349{
350 int i;
351
352 i = netdev_lock_pos(dev->type);
353 lockdep_set_class_and_name(&dev->addr_list_lock,
354 &netdev_addr_lock_key[i],
355 netdev_lock_name[i]);
356}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700357#else
David S. Millercf508b12008-07-22 14:16:42 -0700358static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
359 unsigned short dev_type)
360{
361}
362static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700363{
364}
365#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366
367/*******************************************************************************
368
369 Protocol management and registration routines
370
371*******************************************************************************/
372
373/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 * Add a protocol ID to the list. Now that the input handler is
375 * smarter we can dispense with all the messy stuff that used to be
376 * here.
377 *
378 * BEWARE!!! Protocol handlers, mangling input packets,
379 * MUST BE last in hash buckets and checking protocol handlers
380 * MUST start from promiscuous ptype_all chain in net_bh.
381 * It is true now, do not change it.
382 * Explanation follows: if protocol handler, mangling packet, will
383 * be the first on list, it is not able to sense, that packet
384 * is cloned and should be copied-on-write, so that it will
385 * change it and subsequent readers will get broken packet.
386 * --ANK (980803)
387 */
388
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000389static inline struct list_head *ptype_head(const struct packet_type *pt)
390{
391 if (pt->type == htons(ETH_P_ALL))
392 return &ptype_all;
393 else
394 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
395}
396
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397/**
398 * dev_add_pack - add packet handler
399 * @pt: packet type declaration
400 *
401 * Add a protocol handler to the networking stack. The passed &packet_type
402 * is linked into kernel lists and may not be freed until it has been
403 * removed from the kernel lists.
404 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900405 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 * guarantee all CPU's that are in middle of receiving packets
407 * will see the new packet type (until the next received packet).
408 */
409
410void dev_add_pack(struct packet_type *pt)
411{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000412 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000414 spin_lock(&ptype_lock);
415 list_add_rcu(&pt->list, head);
416 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700418EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420/**
421 * __dev_remove_pack - remove packet handler
422 * @pt: packet type declaration
423 *
424 * Remove a protocol handler that was previously added to the kernel
425 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
426 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900427 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 *
429 * The packet type might still be in use by receivers
430 * and must not be freed until after all the CPU's have gone
431 * through a quiescent state.
432 */
433void __dev_remove_pack(struct packet_type *pt)
434{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000435 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 struct packet_type *pt1;
437
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000438 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439
440 list_for_each_entry(pt1, head, list) {
441 if (pt == pt1) {
442 list_del_rcu(&pt->list);
443 goto out;
444 }
445 }
446
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000447 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000449 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700451EXPORT_SYMBOL(__dev_remove_pack);
452
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453/**
454 * dev_remove_pack - remove packet handler
455 * @pt: packet type declaration
456 *
457 * Remove a protocol handler that was previously added to the kernel
458 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
459 * from the kernel lists and can be freed or reused once this function
460 * returns.
461 *
462 * This call sleeps to guarantee that no CPU is looking at the packet
463 * type after return.
464 */
465void dev_remove_pack(struct packet_type *pt)
466{
467 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900468
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 synchronize_net();
470}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700471EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472
473/******************************************************************************
474
475 Device Boot-time Settings Routines
476
477*******************************************************************************/
478
479/* Boot time configuration table */
480static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
481
482/**
483 * netdev_boot_setup_add - add new setup entry
484 * @name: name of the device
485 * @map: configured settings for the device
486 *
487 * Adds new setup entry to the dev_boot_setup list. The function
488 * returns 0 on error and 1 on success. This is a generic routine to
489 * all netdevices.
490 */
491static int netdev_boot_setup_add(char *name, struct ifmap *map)
492{
493 struct netdev_boot_setup *s;
494 int i;
495
496 s = dev_boot_setup;
497 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
498 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
499 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700500 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 memcpy(&s[i].map, map, sizeof(s[i].map));
502 break;
503 }
504 }
505
506 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
507}
508
509/**
510 * netdev_boot_setup_check - check boot time settings
511 * @dev: the netdevice
512 *
513 * Check boot time settings for the device.
514 * The found settings are set for the device to be used
515 * later in the device probing.
516 * Returns 0 if no settings found, 1 if they are.
517 */
518int netdev_boot_setup_check(struct net_device *dev)
519{
520 struct netdev_boot_setup *s = dev_boot_setup;
521 int i;
522
523 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
524 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700525 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 dev->irq = s[i].map.irq;
527 dev->base_addr = s[i].map.base_addr;
528 dev->mem_start = s[i].map.mem_start;
529 dev->mem_end = s[i].map.mem_end;
530 return 1;
531 }
532 }
533 return 0;
534}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700535EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536
537
538/**
539 * netdev_boot_base - get address from boot time settings
540 * @prefix: prefix for network device
541 * @unit: id for network device
542 *
543 * Check boot time settings for the base address of device.
544 * The found settings are set for the device to be used
545 * later in the device probing.
546 * Returns 0 if no settings found.
547 */
548unsigned long netdev_boot_base(const char *prefix, int unit)
549{
550 const struct netdev_boot_setup *s = dev_boot_setup;
551 char name[IFNAMSIZ];
552 int i;
553
554 sprintf(name, "%s%d", prefix, unit);
555
556 /*
557 * If device already registered then return base of 1
558 * to indicate not to probe for this interface
559 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700560 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 return 1;
562
563 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
564 if (!strcmp(name, s[i].name))
565 return s[i].map.base_addr;
566 return 0;
567}
568
569/*
570 * Saves at boot time configured settings for any netdevice.
571 */
572int __init netdev_boot_setup(char *str)
573{
574 int ints[5];
575 struct ifmap map;
576
577 str = get_options(str, ARRAY_SIZE(ints), ints);
578 if (!str || !*str)
579 return 0;
580
581 /* Save settings */
582 memset(&map, 0, sizeof(map));
583 if (ints[0] > 0)
584 map.irq = ints[1];
585 if (ints[0] > 1)
586 map.base_addr = ints[2];
587 if (ints[0] > 2)
588 map.mem_start = ints[3];
589 if (ints[0] > 3)
590 map.mem_end = ints[4];
591
592 /* Add new entry to the list */
593 return netdev_boot_setup_add(str, &map);
594}
595
596__setup("netdev=", netdev_boot_setup);
597
598/*******************************************************************************
599
600 Device Interface Subroutines
601
602*******************************************************************************/
603
604/**
605 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700606 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 * @name: name to find
608 *
609 * Find an interface by name. Must be called under RTNL semaphore
610 * or @dev_base_lock. If the name is found a pointer to the device
611 * is returned. If the name is not found then %NULL is returned. The
612 * reference counters are not incremented so the caller must be
613 * careful with locks.
614 */
615
Eric W. Biederman881d9662007-09-17 11:56:21 -0700616struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617{
618 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700619 struct net_device *dev;
620 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700622 hlist_for_each_entry(dev, p, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 if (!strncmp(dev->name, name, IFNAMSIZ))
624 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700625
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626 return NULL;
627}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700628EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629
630/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000631 * dev_get_by_name_rcu - find a device by its name
632 * @net: the applicable net namespace
633 * @name: name to find
634 *
635 * Find an interface by name.
636 * If the name is found a pointer to the device is returned.
637 * If the name is not found then %NULL is returned.
638 * The reference counters are not incremented so the caller must be
639 * careful with locks. The caller must hold RCU lock.
640 */
641
642struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
643{
644 struct hlist_node *p;
645 struct net_device *dev;
646 struct hlist_head *head = dev_name_hash(net, name);
647
648 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
649 if (!strncmp(dev->name, name, IFNAMSIZ))
650 return dev;
651
652 return NULL;
653}
654EXPORT_SYMBOL(dev_get_by_name_rcu);
655
656/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700658 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 * @name: name to find
660 *
661 * Find an interface by name. This can be called from any
662 * context and does its own locking. The returned handle has
663 * the usage count incremented and the caller must use dev_put() to
664 * release it when it is no longer needed. %NULL is returned if no
665 * matching device is found.
666 */
667
Eric W. Biederman881d9662007-09-17 11:56:21 -0700668struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669{
670 struct net_device *dev;
671
Eric Dumazet72c95282009-10-30 07:11:27 +0000672 rcu_read_lock();
673 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674 if (dev)
675 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000676 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 return dev;
678}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700679EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680
681/**
682 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700683 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 * @ifindex: index of device
685 *
686 * Search for an interface by index. Returns %NULL if the device
687 * is not found or a pointer to the device. The device has not
688 * had its reference counter increased so the caller must be careful
689 * about locking. The caller must hold either the RTNL semaphore
690 * or @dev_base_lock.
691 */
692
Eric W. Biederman881d9662007-09-17 11:56:21 -0700693struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694{
695 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700696 struct net_device *dev;
697 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700699 hlist_for_each_entry(dev, p, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 if (dev->ifindex == ifindex)
701 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700702
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 return NULL;
704}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700705EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000707/**
708 * dev_get_by_index_rcu - find a device by its ifindex
709 * @net: the applicable net namespace
710 * @ifindex: index of device
711 *
712 * Search for an interface by index. Returns %NULL if the device
713 * is not found or a pointer to the device. The device has not
714 * had its reference counter increased so the caller must be careful
715 * about locking. The caller must hold RCU lock.
716 */
717
718struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
719{
720 struct hlist_node *p;
721 struct net_device *dev;
722 struct hlist_head *head = dev_index_hash(net, ifindex);
723
724 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
725 if (dev->ifindex == ifindex)
726 return dev;
727
728 return NULL;
729}
730EXPORT_SYMBOL(dev_get_by_index_rcu);
731
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732
733/**
734 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700735 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 * @ifindex: index of device
737 *
738 * Search for an interface by index. Returns NULL if the device
739 * is not found or a pointer to the device. The device returned has
740 * had a reference added and the pointer is safe until the user calls
741 * dev_put to indicate they have finished with it.
742 */
743
Eric W. Biederman881d9662007-09-17 11:56:21 -0700744struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745{
746 struct net_device *dev;
747
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000748 rcu_read_lock();
749 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750 if (dev)
751 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000752 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 return dev;
754}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700755EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756
757/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000758 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700759 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760 * @type: media type of device
761 * @ha: hardware address
762 *
763 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800764 * is not found or a pointer to the device.
765 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000766 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 * and the caller must therefore be careful about locking
768 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 */
770
Eric Dumazet941666c2010-12-05 01:23:53 +0000771struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
772 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773{
774 struct net_device *dev;
775
Eric Dumazet941666c2010-12-05 01:23:53 +0000776 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 if (dev->type == type &&
778 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700779 return dev;
780
781 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782}
Eric Dumazet941666c2010-12-05 01:23:53 +0000783EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300784
Eric W. Biederman881d9662007-09-17 11:56:21 -0700785struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700786{
787 struct net_device *dev;
788
789 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700790 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700791 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700792 return dev;
793
794 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700795}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700796EXPORT_SYMBOL(__dev_getfirstbyhwtype);
797
Eric W. Biederman881d9662007-09-17 11:56:21 -0700798struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000800 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000802 rcu_read_lock();
803 for_each_netdev_rcu(net, dev)
804 if (dev->type == type) {
805 dev_hold(dev);
806 ret = dev;
807 break;
808 }
809 rcu_read_unlock();
810 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812EXPORT_SYMBOL(dev_getfirstbyhwtype);
813
814/**
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000815 * dev_get_by_flags_rcu - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700816 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 * @if_flags: IFF_* values
818 * @mask: bitmask of bits in if_flags to check
819 *
820 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000821 * is not found or a pointer to the device. Must be called inside
822 * rcu_read_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 */
824
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000825struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700826 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700828 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829
Pavel Emelianov7562f872007-05-03 15:13:45 -0700830 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800831 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700833 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834 break;
835 }
836 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700837 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838}
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000839EXPORT_SYMBOL(dev_get_by_flags_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840
841/**
842 * dev_valid_name - check if name is okay for network device
843 * @name: name string
844 *
845 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700846 * to allow sysfs to work. We also disallow any kind of
847 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848 */
David S. Miller95f050b2012-03-06 16:12:15 -0500849bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700851 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500852 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700853 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500854 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700855 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500856 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700857
858 while (*name) {
859 if (*name == '/' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -0500860 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700861 name++;
862 }
David S. Miller95f050b2012-03-06 16:12:15 -0500863 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700864}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700865EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866
867/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200868 * __dev_alloc_name - allocate a name for a device
869 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200871 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872 *
873 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700874 * id. It scans list of devices to build up a free map, then chooses
875 * the first empty slot. The caller must hold the dev_base or rtnl lock
876 * while allocating the name and adding the device in order to avoid
877 * duplicates.
878 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
879 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880 */
881
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200882static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883{
884 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885 const char *p;
886 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700887 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888 struct net_device *d;
889
890 p = strnchr(name, IFNAMSIZ-1, '%');
891 if (p) {
892 /*
893 * Verify the string as this thing may have come from
894 * the user. There must be either one "%d" and no other "%"
895 * characters.
896 */
897 if (p[1] != 'd' || strchr(p + 2, '%'))
898 return -EINVAL;
899
900 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700901 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 if (!inuse)
903 return -ENOMEM;
904
Eric W. Biederman881d9662007-09-17 11:56:21 -0700905 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 if (!sscanf(d->name, name, &i))
907 continue;
908 if (i < 0 || i >= max_netdevices)
909 continue;
910
911 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200912 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 if (!strncmp(buf, d->name, IFNAMSIZ))
914 set_bit(i, inuse);
915 }
916
917 i = find_first_zero_bit(inuse, max_netdevices);
918 free_page((unsigned long) inuse);
919 }
920
Octavian Purdilad9031022009-11-18 02:36:59 +0000921 if (buf != name)
922 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200923 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925
926 /* It is possible to run out of possible slots
927 * when the name is long and there isn't enough space left
928 * for the digits, or if all bits are used.
929 */
930 return -ENFILE;
931}
932
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200933/**
934 * dev_alloc_name - allocate a name for a device
935 * @dev: device
936 * @name: name format string
937 *
938 * Passed a format string - eg "lt%d" it will try and find a suitable
939 * id. It scans list of devices to build up a free map, then chooses
940 * the first empty slot. The caller must hold the dev_base or rtnl lock
941 * while allocating the name and adding the device in order to avoid
942 * duplicates.
943 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
944 * Returns the number of the unit assigned or a negative errno code.
945 */
946
947int dev_alloc_name(struct net_device *dev, const char *name)
948{
949 char buf[IFNAMSIZ];
950 struct net *net;
951 int ret;
952
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900953 BUG_ON(!dev_net(dev));
954 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200955 ret = __dev_alloc_name(net, name, buf);
956 if (ret >= 0)
957 strlcpy(dev->name, buf, IFNAMSIZ);
958 return ret;
959}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700960EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200961
Jiri Pirko1c5cae82011-04-30 01:21:32 +0000962static int dev_get_valid_name(struct net_device *dev, const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +0000963{
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000964 struct net *net;
965
966 BUG_ON(!dev_net(dev));
967 net = dev_net(dev);
968
Octavian Purdilad9031022009-11-18 02:36:59 +0000969 if (!dev_valid_name(name))
970 return -EINVAL;
971
Jiri Pirko1c5cae82011-04-30 01:21:32 +0000972 if (strchr(name, '%'))
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000973 return dev_alloc_name(dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +0000974 else if (__dev_get_by_name(net, name))
975 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000976 else if (dev->name != name)
977 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +0000978
979 return 0;
980}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981
982/**
983 * dev_change_name - change name of a device
984 * @dev: device
985 * @newname: name (or format string) must be at least IFNAMSIZ
986 *
987 * Change name of a device, can pass format strings "eth%d".
988 * for wildcarding.
989 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -0700990int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991{
Herbert Xufcc5a032007-07-30 17:03:38 -0700992 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -0700994 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -0700995 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996
997 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900998 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001000 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 if (dev->flags & IFF_UP)
1002 return -EBUSY;
1003
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001004 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
1005 return 0;
1006
Herbert Xufcc5a032007-07-30 17:03:38 -07001007 memcpy(oldname, dev->name, IFNAMSIZ);
1008
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001009 err = dev_get_valid_name(dev, newname);
Octavian Purdilad9031022009-11-18 02:36:59 +00001010 if (err < 0)
1011 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012
Herbert Xufcc5a032007-07-30 17:03:38 -07001013rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001014 ret = device_rename(&dev->dev, dev->name);
1015 if (ret) {
1016 memcpy(dev->name, oldname, IFNAMSIZ);
1017 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001018 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001019
1020 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001021 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001022 write_unlock_bh(&dev_base_lock);
1023
1024 synchronize_rcu();
1025
1026 write_lock_bh(&dev_base_lock);
1027 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001028 write_unlock_bh(&dev_base_lock);
1029
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001030 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001031 ret = notifier_to_errno(ret);
1032
1033 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001034 /* err >= 0 after dev_alloc_name() or stores the first errno */
1035 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001036 err = ret;
1037 memcpy(dev->name, oldname, IFNAMSIZ);
1038 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001039 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001040 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001041 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001042 }
1043 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044
1045 return err;
1046}
1047
1048/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001049 * dev_set_alias - change ifalias of a device
1050 * @dev: device
1051 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001052 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001053 *
1054 * Set ifalias for a device,
1055 */
1056int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1057{
1058 ASSERT_RTNL();
1059
1060 if (len >= IFALIASZ)
1061 return -EINVAL;
1062
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001063 if (!len) {
1064 if (dev->ifalias) {
1065 kfree(dev->ifalias);
1066 dev->ifalias = NULL;
1067 }
1068 return 0;
1069 }
1070
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001071 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001072 if (!dev->ifalias)
1073 return -ENOMEM;
1074
1075 strlcpy(dev->ifalias, alias, len+1);
1076 return len;
1077}
1078
1079
1080/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001081 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001082 * @dev: device to cause notification
1083 *
1084 * Called to indicate a device has changed features.
1085 */
1086void netdev_features_change(struct net_device *dev)
1087{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001088 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001089}
1090EXPORT_SYMBOL(netdev_features_change);
1091
1092/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 * netdev_state_change - device changes state
1094 * @dev: device to cause notification
1095 *
1096 * Called to indicate a device has changed state. This function calls
1097 * the notifier chains for netdev_chain and sends a NEWLINK message
1098 * to the routing socket.
1099 */
1100void netdev_state_change(struct net_device *dev)
1101{
1102 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001103 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1105 }
1106}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001107EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001109int netdev_bonding_change(struct net_device *dev, unsigned long event)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001110{
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001111 return call_netdevice_notifiers(event, dev);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001112}
1113EXPORT_SYMBOL(netdev_bonding_change);
1114
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115/**
1116 * dev_load - load a network module
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001117 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118 * @name: name of interface
1119 *
1120 * If a network interface is not present and the process has suitable
1121 * privileges this function loads the module. If module loading is not
1122 * available in this kernel then it becomes a nop.
1123 */
1124
Eric W. Biederman881d9662007-09-17 11:56:21 -07001125void dev_load(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126{
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001127 struct net_device *dev;
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001128 int no_module;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129
Eric Dumazet72c95282009-10-30 07:11:27 +00001130 rcu_read_lock();
1131 dev = dev_get_by_name_rcu(net, name);
1132 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001133
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001134 no_module = !dev;
1135 if (no_module && capable(CAP_NET_ADMIN))
1136 no_module = request_module("netdev-%s", name);
1137 if (no_module && capable(CAP_SYS_MODULE)) {
1138 if (!request_module("%s", name))
Vinson Lee7cecb522012-06-27 14:32:07 +00001139 pr_warn("Loading kernel module for a network device with CAP_SYS_MODULE (deprecated). Use CAP_NET_ADMIN and alias netdev-%s instead.\n",
1140 name);
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001141 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001143EXPORT_SYMBOL(dev_load);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144
Patrick McHardybd380812010-02-26 06:34:53 +00001145static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001147 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001148 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001149
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001150 ASSERT_RTNL();
1151
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152 if (!netif_device_present(dev))
1153 return -ENODEV;
1154
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001155 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1156 ret = notifier_to_errno(ret);
1157 if (ret)
1158 return ret;
1159
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001161
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001162 if (ops->ndo_validate_addr)
1163 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001164
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001165 if (!ret && ops->ndo_open)
1166 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001167
Jeff Garzikbada3392007-10-23 20:19:37 -07001168 if (ret)
1169 clear_bit(__LINK_STATE_START, &dev->state);
1170 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171 dev->flags |= IFF_UP;
David S. Millerb4bd07c2009-02-06 22:06:43 -08001172 net_dmaengine_get();
Patrick McHardy4417da62007-06-27 01:28:10 -07001173 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 dev_activate(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001176
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177 return ret;
1178}
Patrick McHardybd380812010-02-26 06:34:53 +00001179
1180/**
1181 * dev_open - prepare an interface for use.
1182 * @dev: device to open
1183 *
1184 * Takes a device from down to up state. The device's private open
1185 * function is invoked and then the multicast lists are loaded. Finally
1186 * the device is moved into the up state and a %NETDEV_UP message is
1187 * sent to the netdev notifier chain.
1188 *
1189 * Calling this function on an active interface is a nop. On a failure
1190 * a negative errno code is returned.
1191 */
1192int dev_open(struct net_device *dev)
1193{
1194 int ret;
1195
Patrick McHardybd380812010-02-26 06:34:53 +00001196 if (dev->flags & IFF_UP)
1197 return 0;
1198
Patrick McHardybd380812010-02-26 06:34:53 +00001199 ret = __dev_open(dev);
1200 if (ret < 0)
1201 return ret;
1202
Patrick McHardybd380812010-02-26 06:34:53 +00001203 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1204 call_netdevice_notifiers(NETDEV_UP, dev);
1205
1206 return ret;
1207}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001208EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001209
Octavian Purdila44345722010-12-13 12:44:07 +00001210static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211{
Octavian Purdila44345722010-12-13 12:44:07 +00001212 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001213
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001214 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001215 might_sleep();
1216
Octavian Purdila44345722010-12-13 12:44:07 +00001217 list_for_each_entry(dev, head, unreg_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001218 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219
Octavian Purdila44345722010-12-13 12:44:07 +00001220 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221
Octavian Purdila44345722010-12-13 12:44:07 +00001222 /* Synchronize to scheduled poll. We cannot touch poll list, it
1223 * can be even on different cpu. So just clear netif_running().
1224 *
1225 * dev->stop() will invoke napi_disable() on all of it's
1226 * napi_struct instances on this device.
1227 */
1228 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1229 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230
Octavian Purdila44345722010-12-13 12:44:07 +00001231 dev_deactivate_many(head);
1232
1233 list_for_each_entry(dev, head, unreg_list) {
1234 const struct net_device_ops *ops = dev->netdev_ops;
1235
1236 /*
1237 * Call the device specific close. This cannot fail.
1238 * Only if device is UP
1239 *
1240 * We allow it to be called even after a DETACH hot-plug
1241 * event.
1242 */
1243 if (ops->ndo_stop)
1244 ops->ndo_stop(dev);
1245
Octavian Purdila44345722010-12-13 12:44:07 +00001246 dev->flags &= ~IFF_UP;
Octavian Purdila44345722010-12-13 12:44:07 +00001247 net_dmaengine_put();
1248 }
1249
1250 return 0;
1251}
1252
1253static int __dev_close(struct net_device *dev)
1254{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001255 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001256 LIST_HEAD(single);
1257
1258 list_add(&dev->unreg_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001259 retval = __dev_close_many(&single);
1260 list_del(&single);
1261 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001262}
1263
Eric Dumazet3fbd8752011-01-19 21:23:22 +00001264static int dev_close_many(struct list_head *head)
Octavian Purdila44345722010-12-13 12:44:07 +00001265{
1266 struct net_device *dev, *tmp;
1267 LIST_HEAD(tmp_list);
1268
1269 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1270 if (!(dev->flags & IFF_UP))
1271 list_move(&dev->unreg_list, &tmp_list);
1272
1273 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001274
Octavian Purdila44345722010-12-13 12:44:07 +00001275 list_for_each_entry(dev, head, unreg_list) {
1276 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1277 call_netdevice_notifiers(NETDEV_DOWN, dev);
1278 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001279
Octavian Purdila44345722010-12-13 12:44:07 +00001280 /* rollback_registered_many needs the complete original list */
1281 list_splice(&tmp_list, head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282 return 0;
1283}
Patrick McHardybd380812010-02-26 06:34:53 +00001284
1285/**
1286 * dev_close - shutdown an interface.
1287 * @dev: device to shutdown
1288 *
1289 * This function moves an active device into down state. A
1290 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1291 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1292 * chain.
1293 */
1294int dev_close(struct net_device *dev)
1295{
Eric Dumazete14a5992011-05-10 12:26:06 -07001296 if (dev->flags & IFF_UP) {
1297 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001298
Eric Dumazete14a5992011-05-10 12:26:06 -07001299 list_add(&dev->unreg_list, &single);
1300 dev_close_many(&single);
1301 list_del(&single);
1302 }
Patrick McHardybd380812010-02-26 06:34:53 +00001303 return 0;
1304}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001305EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306
1307
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001308/**
1309 * dev_disable_lro - disable Large Receive Offload on a device
1310 * @dev: device
1311 *
1312 * Disable Large Receive Offload (LRO) on a net device. Must be
1313 * called under RTNL. This is needed if received packets may be
1314 * forwarded to another interface.
1315 */
1316void dev_disable_lro(struct net_device *dev)
1317{
Neil Hormanf11970e2011-05-24 08:31:09 +00001318 /*
1319 * If we're trying to disable lro on a vlan device
1320 * use the underlying physical device instead
1321 */
1322 if (is_vlan_dev(dev))
1323 dev = vlan_dev_real_dev(dev);
1324
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001325 dev->wanted_features &= ~NETIF_F_LRO;
1326 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001327
Michał Mirosław22d59692011-04-21 12:42:15 +00001328 if (unlikely(dev->features & NETIF_F_LRO))
1329 netdev_WARN(dev, "failed to disable LRO!\n");
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001330}
1331EXPORT_SYMBOL(dev_disable_lro);
1332
1333
Eric W. Biederman881d9662007-09-17 11:56:21 -07001334static int dev_boot_phase = 1;
1335
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336/**
1337 * register_netdevice_notifier - register a network notifier block
1338 * @nb: notifier
1339 *
1340 * Register a notifier to be called when network device events occur.
1341 * The notifier passed is linked into the kernel structures and must
1342 * not be reused until it has been unregistered. A negative errno code
1343 * is returned on a failure.
1344 *
1345 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001346 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347 * view of the network device list.
1348 */
1349
1350int register_netdevice_notifier(struct notifier_block *nb)
1351{
1352 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001353 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001354 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355 int err;
1356
1357 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001358 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001359 if (err)
1360 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001361 if (dev_boot_phase)
1362 goto unlock;
1363 for_each_net(net) {
1364 for_each_netdev(net, dev) {
1365 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1366 err = notifier_to_errno(err);
1367 if (err)
1368 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369
Eric W. Biederman881d9662007-09-17 11:56:21 -07001370 if (!(dev->flags & IFF_UP))
1371 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001372
Eric W. Biederman881d9662007-09-17 11:56:21 -07001373 nb->notifier_call(nb, NETDEV_UP, dev);
1374 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001375 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001376
1377unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378 rtnl_unlock();
1379 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001380
1381rollback:
1382 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001383 for_each_net(net) {
1384 for_each_netdev(net, dev) {
1385 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001386 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001387
Eric W. Biederman881d9662007-09-17 11:56:21 -07001388 if (dev->flags & IFF_UP) {
1389 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1390 nb->notifier_call(nb, NETDEV_DOWN, dev);
1391 }
1392 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00001393 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001394 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001395 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001396
RongQing.Li8f891482011-11-30 23:43:07 -05001397outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001398 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001399 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001401EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402
1403/**
1404 * unregister_netdevice_notifier - unregister a network notifier block
1405 * @nb: notifier
1406 *
1407 * Unregister a notifier previously registered by
1408 * register_netdevice_notifier(). The notifier is unlinked into the
1409 * kernel structures and may then be reused. A negative errno code
1410 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001411 *
1412 * After unregistering unregister and down device events are synthesized
1413 * for all devices on the device list to the removed notifier to remove
1414 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415 */
1416
1417int unregister_netdevice_notifier(struct notifier_block *nb)
1418{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001419 struct net_device *dev;
1420 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001421 int err;
1422
1423 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001424 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001425 if (err)
1426 goto unlock;
1427
1428 for_each_net(net) {
1429 for_each_netdev(net, dev) {
1430 if (dev->flags & IFF_UP) {
1431 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1432 nb->notifier_call(nb, NETDEV_DOWN, dev);
1433 }
1434 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1435 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
1436 }
1437 }
1438unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001439 rtnl_unlock();
1440 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001442EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443
1444/**
1445 * call_netdevice_notifiers - call all network notifier blocks
1446 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001447 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448 *
1449 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001450 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001451 */
1452
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001453int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454{
Jiri Pirkoab930472010-04-20 01:45:37 -07001455 ASSERT_RTNL();
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001456 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001458EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001459
Ingo Molnarc5905af2012-02-24 08:31:31 +01001460static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001461#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001462/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001463 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001464 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001465 */
1466static atomic_t netstamp_needed_deferred;
1467#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468
1469void net_enable_timestamp(void)
1470{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001471#ifdef HAVE_JUMP_LABEL
1472 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1473
1474 if (deferred) {
1475 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001476 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001477 return;
1478 }
1479#endif
1480 WARN_ON(in_interrupt());
Ingo Molnarc5905af2012-02-24 08:31:31 +01001481 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001482}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001483EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484
1485void net_disable_timestamp(void)
1486{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001487#ifdef HAVE_JUMP_LABEL
1488 if (in_interrupt()) {
1489 atomic_inc(&netstamp_needed_deferred);
1490 return;
1491 }
1492#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001493 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001495EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001496
Eric Dumazet3b098e22010-05-15 23:57:10 -07001497static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498{
Eric Dumazet588f0332011-11-15 04:12:55 +00001499 skb->tstamp.tv64 = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001500 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001501 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502}
1503
Eric Dumazet588f0332011-11-15 04:12:55 +00001504#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001505 if (static_key_false(&netstamp_needed)) { \
Eric Dumazet588f0332011-11-15 04:12:55 +00001506 if ((COND) && !(SKB)->tstamp.tv64) \
1507 __net_timestamp(SKB); \
1508 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001509
Richard Cochran4dc360c2011-10-19 17:00:35 -04001510static int net_hwtstamp_validate(struct ifreq *ifr)
1511{
1512 struct hwtstamp_config cfg;
1513 enum hwtstamp_tx_types tx_type;
1514 enum hwtstamp_rx_filters rx_filter;
1515 int tx_type_valid = 0;
1516 int rx_filter_valid = 0;
1517
1518 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1519 return -EFAULT;
1520
1521 if (cfg.flags) /* reserved for future extensions */
1522 return -EINVAL;
1523
1524 tx_type = cfg.tx_type;
1525 rx_filter = cfg.rx_filter;
1526
1527 switch (tx_type) {
1528 case HWTSTAMP_TX_OFF:
1529 case HWTSTAMP_TX_ON:
1530 case HWTSTAMP_TX_ONESTEP_SYNC:
1531 tx_type_valid = 1;
1532 break;
1533 }
1534
1535 switch (rx_filter) {
1536 case HWTSTAMP_FILTER_NONE:
1537 case HWTSTAMP_FILTER_ALL:
1538 case HWTSTAMP_FILTER_SOME:
1539 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1540 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1541 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1542 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1543 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1544 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1545 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1546 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1547 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1548 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1549 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1550 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1551 rx_filter_valid = 1;
1552 break;
1553 }
1554
1555 if (!tx_type_valid || !rx_filter_valid)
1556 return -ERANGE;
1557
1558 return 0;
1559}
1560
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001561static inline bool is_skb_forwardable(struct net_device *dev,
1562 struct sk_buff *skb)
1563{
1564 unsigned int len;
1565
1566 if (!(dev->flags & IFF_UP))
1567 return false;
1568
1569 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1570 if (skb->len <= len)
1571 return true;
1572
1573 /* if TSO is enabled, we don't care about the length as the packet
1574 * could be forwarded without being segmented before
1575 */
1576 if (skb_is_gso(skb))
1577 return true;
1578
1579 return false;
1580}
1581
Arnd Bergmann44540962009-11-26 06:07:08 +00001582/**
1583 * dev_forward_skb - loopback an skb to another netif
1584 *
1585 * @dev: destination network device
1586 * @skb: buffer to forward
1587 *
1588 * return values:
1589 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001590 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001591 *
1592 * dev_forward_skb can be used for injecting an skb from the
1593 * start_xmit function of one device into the receive queue
1594 * of another device.
1595 *
1596 * The receiving device may be in another namespace, so
1597 * we have to clear all information in the skb that could
1598 * impact namespace isolation.
1599 */
1600int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1601{
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001602 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1603 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1604 atomic_long_inc(&dev->rx_dropped);
1605 kfree_skb(skb);
1606 return NET_RX_DROP;
1607 }
1608 }
1609
Arnd Bergmann44540962009-11-26 06:07:08 +00001610 skb_orphan(skb);
Ben Greearc736eef2010-07-22 09:54:47 +00001611 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001612
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001613 if (unlikely(!is_skb_forwardable(dev, skb))) {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001614 atomic_long_inc(&dev->rx_dropped);
Eric Dumazet6ec82562010-05-06 00:53:53 -07001615 kfree_skb(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001616 return NET_RX_DROP;
Eric Dumazet6ec82562010-05-06 00:53:53 -07001617 }
Benjamin LaHaise3b9785c2012-03-27 15:55:44 +00001618 skb->skb_iif = 0;
David S. Miller59b99972012-05-10 23:03:34 -04001619 skb->dev = dev;
1620 skb_dst_drop(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001621 skb->tstamp.tv64 = 0;
1622 skb->pkt_type = PACKET_HOST;
1623 skb->protocol = eth_type_trans(skb, dev);
David S. Miller59b99972012-05-10 23:03:34 -04001624 skb->mark = 0;
1625 secpath_reset(skb);
1626 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001627 return netif_rx(skb);
1628}
1629EXPORT_SYMBOL_GPL(dev_forward_skb);
1630
Changli Gao71d9dec2010-12-15 19:57:25 +00001631static inline int deliver_skb(struct sk_buff *skb,
1632 struct packet_type *pt_prev,
1633 struct net_device *orig_dev)
1634{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001635 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1636 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001637 atomic_inc(&skb->users);
1638 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1639}
1640
Linus Torvalds1da177e2005-04-16 15:20:36 -07001641/*
1642 * Support routine. Sends outgoing frames to any network
1643 * taps currently in use.
1644 */
1645
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001646static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647{
1648 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001649 struct sk_buff *skb2 = NULL;
1650 struct packet_type *pt_prev = NULL;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001651
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652 rcu_read_lock();
1653 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1654 /* Never send packets back to the socket
1655 * they originated from - MvS (miquels@drinkel.ow.org)
1656 */
1657 if ((ptype->dev == dev || !ptype->dev) &&
1658 (ptype->af_packet_priv == NULL ||
1659 (struct sock *)ptype->af_packet_priv != skb->sk)) {
Changli Gao71d9dec2010-12-15 19:57:25 +00001660 if (pt_prev) {
1661 deliver_skb(skb2, pt_prev, skb->dev);
1662 pt_prev = ptype;
1663 continue;
1664 }
1665
1666 skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667 if (!skb2)
1668 break;
1669
Eric Dumazet70978182010-12-20 21:22:51 +00001670 net_timestamp_set(skb2);
1671
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672 /* skb->nh should be correctly
1673 set by sender, so that the second statement is
1674 just protection against buggy protocols.
1675 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001676 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001678 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001679 skb2->network_header > skb2->tail) {
Joe Perchese87cc472012-05-13 21:56:26 +00001680 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1681 ntohs(skb2->protocol),
1682 dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001683 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684 }
1685
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001686 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001687 skb2->pkt_type = PACKET_OUTGOING;
Changli Gao71d9dec2010-12-15 19:57:25 +00001688 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689 }
1690 }
Changli Gao71d9dec2010-12-15 19:57:25 +00001691 if (pt_prev)
1692 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001693 rcu_read_unlock();
1694}
1695
Ben Hutchings2c530402012-07-10 10:55:09 +00001696/**
1697 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001698 * @dev: Network device
1699 * @txq: number of queues available
1700 *
1701 * If real_num_tx_queues is changed the tc mappings may no longer be
1702 * valid. To resolve this verify the tc mapping remains valid and if
1703 * not NULL the mapping. With no priorities mapping to this
1704 * offset/count pair it will no longer be used. In the worst case TC0
1705 * is invalid nothing can be done so disable priority mappings. If is
1706 * expected that drivers will fix this mapping if they can before
1707 * calling netif_set_real_num_tx_queues.
1708 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001709static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001710{
1711 int i;
1712 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1713
1714 /* If TC0 is invalidated disable TC mapping */
1715 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001716 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001717 dev->num_tc = 0;
1718 return;
1719 }
1720
1721 /* Invalidated prio to tc mappings set to TC0 */
1722 for (i = 1; i < TC_BITMASK + 1; i++) {
1723 int q = netdev_get_prio_tc_map(dev, i);
1724
1725 tc = &dev->tc_to_txq[q];
1726 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001727 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1728 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001729 netdev_set_prio_tc_map(dev, i, 0);
1730 }
1731 }
1732}
1733
John Fastabendf0796d52010-07-01 13:21:57 +00001734/*
1735 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1736 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1737 */
Tom Herberte6484932010-10-18 18:04:39 +00001738int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00001739{
Tom Herbert1d24eb42010-11-21 13:17:27 +00001740 int rc;
1741
Tom Herberte6484932010-10-18 18:04:39 +00001742 if (txq < 1 || txq > dev->num_tx_queues)
1743 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00001744
Ben Hutchings5c565802011-02-15 19:39:21 +00001745 if (dev->reg_state == NETREG_REGISTERED ||
1746 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00001747 ASSERT_RTNL();
1748
Tom Herbert1d24eb42010-11-21 13:17:27 +00001749 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1750 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00001751 if (rc)
1752 return rc;
1753
John Fastabend4f57c082011-01-17 08:06:04 +00001754 if (dev->num_tc)
1755 netif_setup_tc(dev, txq);
1756
Tom Herberte6484932010-10-18 18:04:39 +00001757 if (txq < dev->real_num_tx_queues)
1758 qdisc_reset_all_tx_gt(dev, txq);
John Fastabendf0796d52010-07-01 13:21:57 +00001759 }
Tom Herberte6484932010-10-18 18:04:39 +00001760
1761 dev->real_num_tx_queues = txq;
1762 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00001763}
1764EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08001765
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001766#ifdef CONFIG_RPS
1767/**
1768 * netif_set_real_num_rx_queues - set actual number of RX queues used
1769 * @dev: Network device
1770 * @rxq: Actual number of RX queues
1771 *
1772 * This must be called either with the rtnl_lock held or before
1773 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07001774 * negative error code. If called before registration, it always
1775 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001776 */
1777int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1778{
1779 int rc;
1780
Tom Herbertbd25fa72010-10-18 18:00:16 +00001781 if (rxq < 1 || rxq > dev->num_rx_queues)
1782 return -EINVAL;
1783
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001784 if (dev->reg_state == NETREG_REGISTERED) {
1785 ASSERT_RTNL();
1786
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001787 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1788 rxq);
1789 if (rc)
1790 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001791 }
1792
1793 dev->real_num_rx_queues = rxq;
1794 return 0;
1795}
1796EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1797#endif
1798
Ben Hutchings2c530402012-07-10 10:55:09 +00001799/**
1800 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00001801 *
1802 * This routine should set an upper limit on the number of RSS queues
1803 * used by default by multiqueue devices.
1804 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00001805int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00001806{
1807 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
1808}
1809EXPORT_SYMBOL(netif_get_num_default_rss_queues);
1810
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001811static inline void __netif_reschedule(struct Qdisc *q)
1812{
1813 struct softnet_data *sd;
1814 unsigned long flags;
1815
1816 local_irq_save(flags);
1817 sd = &__get_cpu_var(softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00001818 q->next_sched = NULL;
1819 *sd->output_queue_tailp = q;
1820 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001821 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1822 local_irq_restore(flags);
1823}
1824
David S. Miller37437bb2008-07-16 02:15:04 -07001825void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08001826{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001827 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1828 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08001829}
1830EXPORT_SYMBOL(__netif_schedule);
1831
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001832void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08001833{
David S. Miller3578b0c2010-08-03 00:24:04 -07001834 if (atomic_dec_and_test(&skb->users)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001835 struct softnet_data *sd;
1836 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08001837
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001838 local_irq_save(flags);
1839 sd = &__get_cpu_var(softnet_data);
1840 skb->next = sd->completion_queue;
1841 sd->completion_queue = skb;
1842 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1843 local_irq_restore(flags);
1844 }
Denis Vlasenko56079432006-03-29 15:57:29 -08001845}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001846EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08001847
1848void dev_kfree_skb_any(struct sk_buff *skb)
1849{
1850 if (in_irq() || irqs_disabled())
1851 dev_kfree_skb_irq(skb);
1852 else
1853 dev_kfree_skb(skb);
1854}
1855EXPORT_SYMBOL(dev_kfree_skb_any);
1856
1857
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001858/**
1859 * netif_device_detach - mark device as removed
1860 * @dev: network device
1861 *
1862 * Mark device as removed from system and therefore no longer available.
1863 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001864void netif_device_detach(struct net_device *dev)
1865{
1866 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1867 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001868 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001869 }
1870}
1871EXPORT_SYMBOL(netif_device_detach);
1872
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001873/**
1874 * netif_device_attach - mark device as attached
1875 * @dev: network device
1876 *
1877 * Mark device as attached from system and restart if needed.
1878 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001879void netif_device_attach(struct net_device *dev)
1880{
1881 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1882 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001883 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001884 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001885 }
1886}
1887EXPORT_SYMBOL(netif_device_attach);
1888
Ben Hutchings36c92472012-01-17 07:57:56 +00001889static void skb_warn_bad_offload(const struct sk_buff *skb)
1890{
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00001891 static const netdev_features_t null_features = 0;
Ben Hutchings36c92472012-01-17 07:57:56 +00001892 struct net_device *dev = skb->dev;
1893 const char *driver = "";
1894
1895 if (dev && dev->dev.parent)
1896 driver = dev_driver_string(dev->dev.parent);
1897
1898 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
1899 "gso_type=%d ip_summed=%d\n",
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00001900 driver, dev ? &dev->features : &null_features,
1901 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00001902 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
1903 skb_shinfo(skb)->gso_type, skb->ip_summed);
1904}
1905
Linus Torvalds1da177e2005-04-16 15:20:36 -07001906/*
1907 * Invalidate hardware checksum when packet is to be mangled, and
1908 * complete checksum manually on outgoing path.
1909 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07001910int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001911{
Al Virod3bc23e2006-11-14 21:24:49 -08001912 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07001913 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001914
Patrick McHardy84fa7932006-08-29 16:44:56 -07001915 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07001916 goto out_set_summed;
1917
1918 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00001919 skb_warn_bad_offload(skb);
1920 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001921 }
1922
Michał Mirosław55508d62010-12-14 15:24:08 +00001923 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07001924 BUG_ON(offset >= skb_headlen(skb));
1925 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1926
1927 offset += skb->csum_offset;
1928 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1929
1930 if (skb_cloned(skb) &&
1931 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001932 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1933 if (ret)
1934 goto out;
1935 }
1936
Herbert Xua0308472007-10-15 01:47:15 -07001937 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07001938out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001939 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001940out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001941 return ret;
1942}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001943EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001944
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001945/**
1946 * skb_gso_segment - Perform segmentation on skb.
1947 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07001948 * @features: features for the output path (see dev->features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001949 *
1950 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07001951 *
1952 * It may return NULL if the skb requires no segmentation. This is
1953 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001954 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00001955struct sk_buff *skb_gso_segment(struct sk_buff *skb,
1956 netdev_features_t features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001957{
1958 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1959 struct packet_type *ptype;
Al Viro252e33462006-11-14 20:48:11 -08001960 __be16 type = skb->protocol;
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001961 int vlan_depth = ETH_HLEN;
Herbert Xua430a432006-07-08 13:34:56 -07001962 int err;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001963
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001964 while (type == htons(ETH_P_8021Q)) {
1965 struct vlan_hdr *vh;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001966
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001967 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
Jesse Gross7b9c6092010-10-20 13:56:04 +00001968 return ERR_PTR(-EINVAL);
1969
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001970 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
1971 type = vh->h_vlan_encapsulated_proto;
1972 vlan_depth += VLAN_HLEN;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001973 }
1974
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001975 skb_reset_mac_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001976 skb->mac_len = skb->network_header - skb->mac_header;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001977 __skb_pull(skb, skb->mac_len);
1978
Herbert Xu67fd1a72009-01-19 16:26:44 -08001979 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00001980 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08001981
Herbert Xua430a432006-07-08 13:34:56 -07001982 if (skb_header_cloned(skb) &&
1983 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1984 return ERR_PTR(err);
1985 }
1986
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001987 rcu_read_lock();
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08001988 list_for_each_entry_rcu(ptype,
1989 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001990 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07001991 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Herbert Xua430a432006-07-08 13:34:56 -07001992 err = ptype->gso_send_check(skb);
1993 segs = ERR_PTR(err);
1994 if (err || skb_gso_ok(skb, features))
1995 break;
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001996 __skb_push(skb, (skb->data -
1997 skb_network_header(skb)));
Herbert Xua430a432006-07-08 13:34:56 -07001998 }
Herbert Xu576a30e2006-06-27 13:22:38 -07001999 segs = ptype->gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002000 break;
2001 }
2002 }
2003 rcu_read_unlock();
2004
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07002005 __skb_push(skb, skb->data - skb_mac_header(skb));
Herbert Xu576a30e2006-06-27 13:22:38 -07002006
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002007 return segs;
2008}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002009EXPORT_SYMBOL(skb_gso_segment);
2010
Herbert Xufb286bb2005-11-10 13:01:24 -08002011/* Take action when hardware reception checksum errors are detected. */
2012#ifdef CONFIG_BUG
2013void netdev_rx_csum_fault(struct net_device *dev)
2014{
2015 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002016 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002017 dump_stack();
2018 }
2019}
2020EXPORT_SYMBOL(netdev_rx_csum_fault);
2021#endif
2022
Linus Torvalds1da177e2005-04-16 15:20:36 -07002023/* Actually, we should eliminate this check as soon as we know, that:
2024 * 1. IOMMU is present and allows to map all the memory.
2025 * 2. No high memory really exists on this machine.
2026 */
2027
Eric Dumazet9092c652010-04-02 13:34:49 -07002028static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002030#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002031 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002032 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002033 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2034 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2035 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002036 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002037 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002038 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002039
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002040 if (PCI_DMA_BUS_IS_PHYS) {
2041 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002042
Eric Dumazet9092c652010-04-02 13:34:49 -07002043 if (!pdev)
2044 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002045 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002046 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2047 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002048 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2049 return 1;
2050 }
2051 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002052#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002053 return 0;
2054}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002055
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002056struct dev_gso_cb {
2057 void (*destructor)(struct sk_buff *skb);
2058};
2059
2060#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
2061
2062static void dev_gso_skb_destructor(struct sk_buff *skb)
2063{
2064 struct dev_gso_cb *cb;
2065
2066 do {
2067 struct sk_buff *nskb = skb->next;
2068
2069 skb->next = nskb->next;
2070 nskb->next = NULL;
2071 kfree_skb(nskb);
2072 } while (skb->next);
2073
2074 cb = DEV_GSO_CB(skb);
2075 if (cb->destructor)
2076 cb->destructor(skb);
2077}
2078
2079/**
2080 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2081 * @skb: buffer to segment
Jesse Gross91ecb632011-01-09 06:23:33 +00002082 * @features: device features as applicable to this skb
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002083 *
2084 * This function segments the given skb and stores the list of segments
2085 * in skb->next.
2086 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002087static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002088{
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002089 struct sk_buff *segs;
2090
Herbert Xu576a30e2006-06-27 13:22:38 -07002091 segs = skb_gso_segment(skb, features);
2092
2093 /* Verifying header integrity only. */
2094 if (!segs)
2095 return 0;
2096
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07002097 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002098 return PTR_ERR(segs);
2099
2100 skb->next = segs;
2101 DEV_GSO_CB(skb)->destructor = skb->destructor;
2102 skb->destructor = dev_gso_skb_destructor;
2103
2104 return 0;
2105}
2106
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002107static bool can_checksum_protocol(netdev_features_t features, __be16 protocol)
Jesse Gross03634662011-01-09 06:23:35 +00002108{
2109 return ((features & NETIF_F_GEN_CSUM) ||
2110 ((features & NETIF_F_V4_CSUM) &&
2111 protocol == htons(ETH_P_IP)) ||
2112 ((features & NETIF_F_V6_CSUM) &&
2113 protocol == htons(ETH_P_IPV6)) ||
2114 ((features & NETIF_F_FCOE_CRC) &&
2115 protocol == htons(ETH_P_FCOE)));
2116}
2117
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002118static netdev_features_t harmonize_features(struct sk_buff *skb,
2119 __be16 protocol, netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002120{
Eric Dumazetd4027862011-01-19 00:51:36 +00002121 if (!can_checksum_protocol(features, protocol)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002122 features &= ~NETIF_F_ALL_CSUM;
2123 features &= ~NETIF_F_SG;
2124 } else if (illegal_highdma(skb->dev, skb)) {
2125 features &= ~NETIF_F_SG;
2126 }
2127
2128 return features;
2129}
2130
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002131netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002132{
2133 __be16 protocol = skb->protocol;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002134 netdev_features_t features = skb->dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002135
2136 if (protocol == htons(ETH_P_8021Q)) {
2137 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2138 protocol = veh->h_vlan_encapsulated_proto;
Jesse Grossf01a5232011-01-09 06:23:31 +00002139 } else if (!vlan_tx_tag_present(skb)) {
2140 return harmonize_features(skb, protocol, features);
2141 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002142
Jesse Gross6ee400a2011-01-17 20:46:00 +00002143 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
Jesse Grossf01a5232011-01-09 06:23:31 +00002144
2145 if (protocol != htons(ETH_P_8021Q)) {
2146 return harmonize_features(skb, protocol, features);
2147 } else {
2148 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
Jesse Gross6ee400a2011-01-17 20:46:00 +00002149 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
Jesse Grossf01a5232011-01-09 06:23:31 +00002150 return harmonize_features(skb, protocol, features);
2151 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002152}
Jesse Grossf01a5232011-01-09 06:23:31 +00002153EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002154
John Fastabend6afff0c2010-06-16 14:18:12 +00002155/*
2156 * Returns true if either:
2157 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2158 * 2. skb is fragmented and the device does not support SG, or if
2159 * at least one of fragments is in highmem and device does not
2160 * support DMA from it.
2161 */
2162static inline int skb_needs_linearize(struct sk_buff *skb,
Jesse Gross02932ce2011-01-09 06:23:34 +00002163 int features)
John Fastabend6afff0c2010-06-16 14:18:12 +00002164{
Jesse Gross02932ce2011-01-09 06:23:34 +00002165 return skb_is_nonlinear(skb) &&
2166 ((skb_has_frag_list(skb) &&
2167 !(features & NETIF_F_FRAGLIST)) ||
Jesse Grosse1e78db2010-10-29 12:14:53 +00002168 (skb_shinfo(skb)->nr_frags &&
Jesse Gross02932ce2011-01-09 06:23:34 +00002169 !(features & NETIF_F_SG)));
John Fastabend6afff0c2010-06-16 14:18:12 +00002170}
2171
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002172int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2173 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002174{
Stephen Hemminger00829822008-11-20 20:14:53 -08002175 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00002176 int rc = NETDEV_TX_OK;
Koki Sanagiec764bf2011-05-30 21:48:34 +00002177 unsigned int skb_len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002178
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002179 if (likely(!skb->next)) {
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002180 netdev_features_t features;
Jesse Grossfc741212011-01-09 06:23:32 +00002181
Eric Dumazet93f154b2009-05-18 22:19:19 -07002182 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002183 * If device doesn't need skb->dst, release it right now while
Eric Dumazet93f154b2009-05-18 22:19:19 -07002184 * its hot in this cpu cache
2185 */
Eric Dumazetadf30902009-06-02 05:19:30 +00002186 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2187 skb_dst_drop(skb);
2188
Eric Dumazet15c2d75f2010-12-07 00:30:37 +00002189 if (!list_empty(&ptype_all))
2190 dev_queue_xmit_nit(skb, dev);
2191
Jesse Grossfc741212011-01-09 06:23:32 +00002192 features = netif_skb_features(skb);
2193
Jesse Gross7b9c6092010-10-20 13:56:04 +00002194 if (vlan_tx_tag_present(skb) &&
Jesse Grossfc741212011-01-09 06:23:32 +00002195 !(features & NETIF_F_HW_VLAN_TX)) {
Jesse Gross7b9c6092010-10-20 13:56:04 +00002196 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2197 if (unlikely(!skb))
2198 goto out;
2199
2200 skb->vlan_tci = 0;
2201 }
2202
Jesse Grossfc741212011-01-09 06:23:32 +00002203 if (netif_needs_gso(skb, features)) {
Jesse Gross91ecb632011-01-09 06:23:33 +00002204 if (unlikely(dev_gso_segment(skb, features)))
David S. Miller9ccb8972010-04-22 01:02:07 -07002205 goto out_kfree_skb;
2206 if (skb->next)
2207 goto gso;
John Fastabend6afff0c2010-06-16 14:18:12 +00002208 } else {
Jesse Gross02932ce2011-01-09 06:23:34 +00002209 if (skb_needs_linearize(skb, features) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002210 __skb_linearize(skb))
2211 goto out_kfree_skb;
2212
2213 /* If packet is not checksummed and device does not
2214 * support checksumming for this protocol, complete
2215 * checksumming here.
2216 */
2217 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Michał Mirosław55508d62010-12-14 15:24:08 +00002218 skb_set_transport_header(skb,
2219 skb_checksum_start_offset(skb));
Jesse Gross03634662011-01-09 06:23:35 +00002220 if (!(features & NETIF_F_ALL_CSUM) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002221 skb_checksum_help(skb))
2222 goto out_kfree_skb;
2223 }
David S. Miller9ccb8972010-04-22 01:02:07 -07002224 }
2225
Koki Sanagiec764bf2011-05-30 21:48:34 +00002226 skb_len = skb->len;
Patrick Ohlyac45f602009-02-12 05:03:37 +00002227 rc = ops->ndo_start_xmit(skb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002228 trace_net_dev_xmit(skb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002229 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07002230 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00002231 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002232 }
2233
Herbert Xu576a30e2006-06-27 13:22:38 -07002234gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002235 do {
2236 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002237
2238 skb->next = nskb->next;
2239 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00002240
2241 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002242 * If device doesn't need nskb->dst, release it right now while
Krishna Kumar068a2de2009-12-09 20:59:58 +00002243 * its hot in this cpu cache
2244 */
2245 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2246 skb_dst_drop(nskb);
2247
Koki Sanagiec764bf2011-05-30 21:48:34 +00002248 skb_len = nskb->len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002249 rc = ops->ndo_start_xmit(nskb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002250 trace_net_dev_xmit(nskb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002251 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002252 if (rc & ~NETDEV_TX_MASK)
2253 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07002254 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002255 skb->next = nskb;
2256 return rc;
2257 }
Eric Dumazet08baf562009-05-25 22:58:01 -07002258 txq_trans_update(txq);
Tom Herbert734664982011-11-28 16:32:44 +00002259 if (unlikely(netif_xmit_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07002260 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002261 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002262
Patrick McHardy572a9d72009-11-10 06:14:14 +00002263out_kfree_gso_skb:
2264 if (likely(skb->next == NULL))
2265 skb->destructor = DEV_GSO_CB(skb)->destructor;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002266out_kfree_skb:
2267 kfree_skb(skb);
Jesse Gross7b9c6092010-10-20 13:56:04 +00002268out:
Patrick McHardy572a9d72009-11-10 06:14:14 +00002269 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002270}
2271
Tom Herbert0a9627f2010-03-16 08:03:29 +00002272static u32 hashrnd __read_mostly;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002273
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002274/*
2275 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2276 * to be used as a distribution range.
2277 */
2278u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2279 unsigned int num_tx_queues)
David S. Miller8f0f2222008-07-15 03:47:03 -07002280{
David S. Miller70192982009-01-27 16:34:47 -08002281 u32 hash;
John Fastabend4f57c082011-01-17 08:06:04 +00002282 u16 qoffset = 0;
2283 u16 qcount = num_tx_queues;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002284
David S. Miller513de112009-05-03 14:43:10 -07002285 if (skb_rx_queue_recorded(skb)) {
2286 hash = skb_get_rx_queue(skb);
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002287 while (unlikely(hash >= num_tx_queues))
2288 hash -= num_tx_queues;
David S. Miller513de112009-05-03 14:43:10 -07002289 return hash;
2290 }
Eric Dumazetec581f62009-05-01 09:05:06 -07002291
John Fastabend4f57c082011-01-17 08:06:04 +00002292 if (dev->num_tc) {
2293 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2294 qoffset = dev->tc_to_txq[tc].offset;
2295 qcount = dev->tc_to_txq[tc].count;
2296 }
2297
Eric Dumazetec581f62009-05-01 09:05:06 -07002298 if (skb->sk && skb->sk->sk_hash)
David S. Miller70192982009-01-27 16:34:47 -08002299 hash = skb->sk->sk_hash;
Eric Dumazetec581f62009-05-01 09:05:06 -07002300 else
Eric Dumazet62b1a8a2012-06-14 06:42:44 +00002301 hash = (__force u16) skb->protocol;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002302 hash = jhash_1word(hash, hashrnd);
David S. Millerd5a9e242009-01-27 16:22:11 -08002303
John Fastabend4f57c082011-01-17 08:06:04 +00002304 return (u16) (((u64) hash * qcount) >> 32) + qoffset;
David S. Miller8f0f2222008-07-15 03:47:03 -07002305}
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002306EXPORT_SYMBOL(__skb_tx_hash);
David S. Miller8f0f2222008-07-15 03:47:03 -07002307
Eric Dumazeted046422009-11-13 21:54:04 +00002308static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2309{
2310 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
Joe Perchese87cc472012-05-13 21:56:26 +00002311 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
2312 dev->name, queue_index,
2313 dev->real_num_tx_queues);
Eric Dumazeted046422009-11-13 21:54:04 +00002314 return 0;
2315 }
2316 return queue_index;
2317}
2318
Tom Herbert1d24eb42010-11-21 13:17:27 +00002319static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2320{
Tom Herbertbf264142010-11-26 08:36:09 +00002321#ifdef CONFIG_XPS
Tom Herbert1d24eb42010-11-21 13:17:27 +00002322 struct xps_dev_maps *dev_maps;
2323 struct xps_map *map;
2324 int queue_index = -1;
2325
2326 rcu_read_lock();
2327 dev_maps = rcu_dereference(dev->xps_maps);
2328 if (dev_maps) {
2329 map = rcu_dereference(
2330 dev_maps->cpu_map[raw_smp_processor_id()]);
2331 if (map) {
2332 if (map->len == 1)
2333 queue_index = map->queues[0];
2334 else {
2335 u32 hash;
2336 if (skb->sk && skb->sk->sk_hash)
2337 hash = skb->sk->sk_hash;
2338 else
2339 hash = (__force u16) skb->protocol ^
2340 skb->rxhash;
2341 hash = jhash_1word(hash, hashrnd);
2342 queue_index = map->queues[
2343 ((u64)hash * map->len) >> 32];
2344 }
2345 if (unlikely(queue_index >= dev->real_num_tx_queues))
2346 queue_index = -1;
2347 }
2348 }
2349 rcu_read_unlock();
2350
2351 return queue_index;
2352#else
2353 return -1;
2354#endif
2355}
2356
David S. Millere8a04642008-07-17 00:34:19 -07002357static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2358 struct sk_buff *skb)
2359{
Tom Herbertb0f77d02010-07-14 20:50:29 -07002360 int queue_index;
Helmut Schaadeabc772010-09-03 02:39:56 +00002361 const struct net_device_ops *ops = dev->netdev_ops;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002362
Tom Herbert3853b582010-11-21 13:17:29 +00002363 if (dev->real_num_tx_queues == 1)
2364 queue_index = 0;
2365 else if (ops->ndo_select_queue) {
Helmut Schaadeabc772010-09-03 02:39:56 +00002366 queue_index = ops->ndo_select_queue(dev, skb);
2367 queue_index = dev_cap_txqueue(dev, queue_index);
2368 } else {
2369 struct sock *sk = skb->sk;
2370 queue_index = sk_tx_queue_get(sk);
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002371
Tom Herbert3853b582010-11-21 13:17:29 +00002372 if (queue_index < 0 || skb->ooo_okay ||
2373 queue_index >= dev->real_num_tx_queues) {
2374 int old_index = queue_index;
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002375
Tom Herbert1d24eb42010-11-21 13:17:27 +00002376 queue_index = get_xps_queue(dev, skb);
2377 if (queue_index < 0)
2378 queue_index = skb_tx_hash(dev, skb);
Tom Herbert3853b582010-11-21 13:17:29 +00002379
2380 if (queue_index != old_index && sk) {
2381 struct dst_entry *dst =
2382 rcu_dereference_check(sk->sk_dst_cache, 1);
Eric Dumazet8728c542010-04-11 21:18:17 +00002383
2384 if (dst && skb_dst(skb) == dst)
2385 sk_tx_queue_set(sk, queue_index);
2386 }
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002387 }
2388 }
David S. Millereae792b2008-07-15 03:03:33 -07002389
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002390 skb_set_queue_mapping(skb, queue_index);
2391 return netdev_get_tx_queue(dev, queue_index);
David S. Millere8a04642008-07-17 00:34:19 -07002392}
2393
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002394static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2395 struct net_device *dev,
2396 struct netdev_queue *txq)
2397{
2398 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002399 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002400 int rc;
2401
Eric Dumazeta2da5702011-01-20 03:48:19 +00002402 qdisc_skb_cb(skb)->pkt_len = skb->len;
2403 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002404 /*
2405 * Heuristic to force contended enqueues to serialize on a
2406 * separate lock before trying to get qdisc main lock.
2407 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2408 * and dequeue packets faster.
2409 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002410 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002411 if (unlikely(contended))
2412 spin_lock(&q->busylock);
2413
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002414 spin_lock(root_lock);
2415 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2416 kfree_skb(skb);
2417 rc = NET_XMIT_DROP;
2418 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002419 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002420 /*
2421 * This is a work-conserving queue; there are no old skbs
2422 * waiting to be sent out; and the qdisc is not running -
2423 * xmit the skb directly.
2424 */
Eric Dumazet7fee2262010-05-11 23:19:48 +00002425 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2426 skb_dst_force(skb);
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002427
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002428 qdisc_bstats_update(q, skb);
2429
Eric Dumazet79640a42010-06-02 05:09:29 -07002430 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2431 if (unlikely(contended)) {
2432 spin_unlock(&q->busylock);
2433 contended = false;
2434 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002435 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002436 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002437 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002438
2439 rc = NET_XMIT_SUCCESS;
2440 } else {
Eric Dumazet7fee2262010-05-11 23:19:48 +00002441 skb_dst_force(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002442 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002443 if (qdisc_run_begin(q)) {
2444 if (unlikely(contended)) {
2445 spin_unlock(&q->busylock);
2446 contended = false;
2447 }
2448 __qdisc_run(q);
2449 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002450 }
2451 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002452 if (unlikely(contended))
2453 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002454 return rc;
2455}
2456
Neil Horman5bc14212011-11-22 05:10:51 +00002457#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
2458static void skb_update_prio(struct sk_buff *skb)
2459{
Igor Maravic6977a792011-11-25 07:44:54 +00002460 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00002461
Eric Dumazet91c68ce2012-07-08 21:45:10 +00002462 if (!skb->priority && skb->sk && map) {
2463 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2464
2465 if (prioidx < map->priomap_len)
2466 skb->priority = map->priomap[prioidx];
2467 }
Neil Horman5bc14212011-11-22 05:10:51 +00002468}
2469#else
2470#define skb_update_prio(skb)
2471#endif
2472
Eric Dumazet745e20f2010-09-29 13:23:09 -07002473static DEFINE_PER_CPU(int, xmit_recursion);
David S. Miller11a766c2010-10-25 12:51:55 -07002474#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002475
Dave Jonesd29f7492008-07-22 14:09:06 -07002476/**
Michel Machado95603e22012-06-12 10:16:35 +00002477 * dev_loopback_xmit - loop back @skb
2478 * @skb: buffer to transmit
2479 */
2480int dev_loopback_xmit(struct sk_buff *skb)
2481{
2482 skb_reset_mac_header(skb);
2483 __skb_pull(skb, skb_network_offset(skb));
2484 skb->pkt_type = PACKET_LOOPBACK;
2485 skb->ip_summed = CHECKSUM_UNNECESSARY;
2486 WARN_ON(!skb_dst(skb));
2487 skb_dst_force(skb);
2488 netif_rx_ni(skb);
2489 return 0;
2490}
2491EXPORT_SYMBOL(dev_loopback_xmit);
2492
2493/**
Dave Jonesd29f7492008-07-22 14:09:06 -07002494 * dev_queue_xmit - transmit a buffer
2495 * @skb: buffer to transmit
2496 *
2497 * Queue a buffer for transmission to a network device. The caller must
2498 * have set the device and priority and built the buffer before calling
2499 * this function. The function can be called from an interrupt.
2500 *
2501 * A negative errno code is returned on a failure. A success does not
2502 * guarantee the frame will be transmitted as it may be dropped due
2503 * to congestion or traffic shaping.
2504 *
2505 * -----------------------------------------------------------------------------------
2506 * I notice this method can also return errors from the queue disciplines,
2507 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2508 * be positive.
2509 *
2510 * Regardless of the return value, the skb is consumed, so it is currently
2511 * difficult to retry a send to this method. (You can bump the ref count
2512 * before sending to hold a reference for retry if you are careful.)
2513 *
2514 * When calling this method, interrupts MUST be enabled. This is because
2515 * the BH enable code must have IRQs enabled so that it will not deadlock.
2516 * --BLG
2517 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002518int dev_queue_xmit(struct sk_buff *skb)
2519{
2520 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002521 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002522 struct Qdisc *q;
2523 int rc = -ENOMEM;
2524
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002525 /* Disable soft irqs for various locks below. Also
2526 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002527 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002528 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002529
Neil Horman5bc14212011-11-22 05:10:51 +00002530 skb_update_prio(skb);
2531
David S. Millereae792b2008-07-15 03:03:33 -07002532 txq = dev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002533 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002534
Linus Torvalds1da177e2005-04-16 15:20:36 -07002535#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002536 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002537#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002538 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002539 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002540 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002541 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002542 }
2543
2544 /* The device has no queue. Common case for software devices:
2545 loopback, all the sorts of tunnels...
2546
Herbert Xu932ff272006-06-09 12:20:56 -07002547 Really, it is unlikely that netif_tx_lock protection is necessary
2548 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002549 counters.)
2550 However, it is possible, that they rely on protection
2551 made by us here.
2552
2553 Check this and shot the lock. It is not prone from deadlocks.
2554 Either shot noqueue qdisc, it is even simpler 8)
2555 */
2556 if (dev->flags & IFF_UP) {
2557 int cpu = smp_processor_id(); /* ok because BHs are off */
2558
David S. Millerc773e842008-07-08 23:13:53 -07002559 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002560
Eric Dumazet745e20f2010-09-29 13:23:09 -07002561 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2562 goto recursion_alert;
2563
David S. Millerc773e842008-07-08 23:13:53 -07002564 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002565
Tom Herbert734664982011-11-28 16:32:44 +00002566 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002567 __this_cpu_inc(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002568 rc = dev_hard_start_xmit(skb, dev, txq);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002569 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002570 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002571 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002572 goto out;
2573 }
2574 }
David S. Millerc773e842008-07-08 23:13:53 -07002575 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00002576 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
2577 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002578 } else {
2579 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07002580 * unfortunately
2581 */
2582recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00002583 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
2584 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002585 }
2586 }
2587
2588 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002589 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002590
Linus Torvalds1da177e2005-04-16 15:20:36 -07002591 kfree_skb(skb);
2592 return rc;
2593out:
Herbert Xud4828d82006-06-22 02:28:18 -07002594 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002595 return rc;
2596}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002597EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002598
2599
2600/*=======================================================================
2601 Receiver routines
2602 =======================================================================*/
2603
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002604int netdev_max_backlog __read_mostly = 1000;
Eric Dumazet3b098e22010-05-15 23:57:10 -07002605int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002606int netdev_budget __read_mostly = 300;
2607int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002608
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002609/* Called with irq disabled */
2610static inline void ____napi_schedule(struct softnet_data *sd,
2611 struct napi_struct *napi)
2612{
2613 list_add_tail(&napi->poll_list, &sd->poll_list);
2614 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2615}
2616
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002617/*
2618 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
Tom Herbertbdeab992011-08-14 19:45:55 +00002619 * and src/dst port numbers. Sets rxhash in skb to non-zero hash value
2620 * on success, zero indicates no valid hash. Also, sets l4_rxhash in skb
2621 * if hash is a canonical 4-tuple hash over transport ports.
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002622 */
Tom Herbertbdeab992011-08-14 19:45:55 +00002623void __skb_get_rxhash(struct sk_buff *skb)
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002624{
Eric Dumazet4504b862011-11-28 05:23:23 +00002625 struct flow_keys keys;
2626 u32 hash;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002627
Eric Dumazet4504b862011-11-28 05:23:23 +00002628 if (!skb_flow_dissect(skb, &keys))
2629 return;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002630
Eric Dumazet4504b862011-11-28 05:23:23 +00002631 if (keys.ports) {
2632 if ((__force u16)keys.port16[1] < (__force u16)keys.port16[0])
2633 swap(keys.port16[0], keys.port16[1]);
2634 skb->l4_rxhash = 1;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002635 }
2636
2637 /* get a consistent hash (same value on both flow directions) */
Eric Dumazet4504b862011-11-28 05:23:23 +00002638 if ((__force u32)keys.dst < (__force u32)keys.src)
2639 swap(keys.dst, keys.src);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002640
Eric Dumazet4504b862011-11-28 05:23:23 +00002641 hash = jhash_3words((__force u32)keys.dst,
2642 (__force u32)keys.src,
2643 (__force u32)keys.ports, hashrnd);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002644 if (!hash)
2645 hash = 1;
2646
Tom Herbertbdeab992011-08-14 19:45:55 +00002647 skb->rxhash = hash;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002648}
2649EXPORT_SYMBOL(__skb_get_rxhash);
2650
Eric Dumazetdf334542010-03-24 19:13:54 +00002651#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07002652
2653/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002654struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07002655EXPORT_SYMBOL(rps_sock_flow_table);
2656
Ingo Molnarc5905af2012-02-24 08:31:31 +01002657struct static_key rps_needed __read_mostly;
Eric Dumazetadc93002011-11-17 03:13:26 +00002658
Ben Hutchingsc4454772011-01-19 11:03:53 +00002659static struct rps_dev_flow *
2660set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2661 struct rps_dev_flow *rflow, u16 next_cpu)
2662{
Ben Hutchings09994d12011-10-03 04:42:46 +00002663 if (next_cpu != RPS_NO_CPU) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00002664#ifdef CONFIG_RFS_ACCEL
2665 struct netdev_rx_queue *rxqueue;
2666 struct rps_dev_flow_table *flow_table;
2667 struct rps_dev_flow *old_rflow;
2668 u32 flow_id;
2669 u16 rxq_index;
2670 int rc;
2671
2672 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00002673 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2674 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00002675 goto out;
2676 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2677 if (rxq_index == skb_get_rx_queue(skb))
2678 goto out;
2679
2680 rxqueue = dev->_rx + rxq_index;
2681 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2682 if (!flow_table)
2683 goto out;
2684 flow_id = skb->rxhash & flow_table->mask;
2685 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2686 rxq_index, flow_id);
2687 if (rc < 0)
2688 goto out;
2689 old_rflow = rflow;
2690 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00002691 rflow->filter = rc;
2692 if (old_rflow->filter == rflow->filter)
2693 old_rflow->filter = RPS_NO_FILTER;
2694 out:
2695#endif
2696 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00002697 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002698 }
2699
Ben Hutchings09994d12011-10-03 04:42:46 +00002700 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002701 return rflow;
2702}
2703
Tom Herbert0a9627f2010-03-16 08:03:29 +00002704/*
2705 * get_rps_cpu is called from netif_receive_skb and returns the target
2706 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002707 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00002708 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002709static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2710 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002711{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002712 struct netdev_rx_queue *rxqueue;
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002713 struct rps_map *map;
Tom Herbertfec5e652010-04-16 16:01:27 -07002714 struct rps_dev_flow_table *flow_table;
2715 struct rps_sock_flow_table *sock_flow_table;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002716 int cpu = -1;
Tom Herbertfec5e652010-04-16 16:01:27 -07002717 u16 tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002718
Tom Herbert0a9627f2010-03-16 08:03:29 +00002719 if (skb_rx_queue_recorded(skb)) {
2720 u16 index = skb_get_rx_queue(skb);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002721 if (unlikely(index >= dev->real_num_rx_queues)) {
2722 WARN_ONCE(dev->real_num_rx_queues > 1,
2723 "%s received packet on queue %u, but number "
2724 "of RX queues is %u\n",
2725 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002726 goto done;
2727 }
2728 rxqueue = dev->_rx + index;
2729 } else
2730 rxqueue = dev->_rx;
2731
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002732 map = rcu_dereference(rxqueue->rps_map);
2733 if (map) {
Tom Herbert85875232011-01-31 16:23:42 -08002734 if (map->len == 1 &&
Eric Dumazet33d480c2011-08-11 19:30:52 +00002735 !rcu_access_pointer(rxqueue->rps_flow_table)) {
Changli Gao6febfca2010-09-03 23:12:37 +00002736 tcpu = map->cpus[0];
2737 if (cpu_online(tcpu))
2738 cpu = tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002739 goto done;
Eric Dumazetb249dcb2010-04-19 21:56:38 +00002740 }
Eric Dumazet33d480c2011-08-11 19:30:52 +00002741 } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002742 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002743 }
2744
Changli Gao2d47b452010-08-17 19:00:56 +00002745 skb_reset_network_header(skb);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002746 if (!skb_get_rxhash(skb))
Tom Herbert0a9627f2010-03-16 08:03:29 +00002747 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002748
Tom Herbertfec5e652010-04-16 16:01:27 -07002749 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2750 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2751 if (flow_table && sock_flow_table) {
2752 u16 next_cpu;
2753 struct rps_dev_flow *rflow;
2754
2755 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2756 tcpu = rflow->cpu;
2757
2758 next_cpu = sock_flow_table->ents[skb->rxhash &
2759 sock_flow_table->mask];
2760
2761 /*
2762 * If the desired CPU (where last recvmsg was done) is
2763 * different from current CPU (one in the rx-queue flow
2764 * table entry), switch if one of the following holds:
2765 * - Current CPU is unset (equal to RPS_NO_CPU).
2766 * - Current CPU is offline.
2767 * - The current CPU's queue tail has advanced beyond the
2768 * last packet that was enqueued using this table entry.
2769 * This guarantees that all previous packets for the flow
2770 * have been dequeued, thus preserving in order delivery.
2771 */
2772 if (unlikely(tcpu != next_cpu) &&
2773 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2774 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Ben Hutchingsc4454772011-01-19 11:03:53 +00002775 rflow->last_qtail)) >= 0))
2776 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
2777
Tom Herbertfec5e652010-04-16 16:01:27 -07002778 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2779 *rflowp = rflow;
2780 cpu = tcpu;
2781 goto done;
2782 }
2783 }
2784
Tom Herbert0a9627f2010-03-16 08:03:29 +00002785 if (map) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002786 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
Tom Herbert0a9627f2010-03-16 08:03:29 +00002787
2788 if (cpu_online(tcpu)) {
2789 cpu = tcpu;
2790 goto done;
2791 }
2792 }
2793
2794done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00002795 return cpu;
2796}
2797
Ben Hutchingsc4454772011-01-19 11:03:53 +00002798#ifdef CONFIG_RFS_ACCEL
2799
2800/**
2801 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
2802 * @dev: Device on which the filter was set
2803 * @rxq_index: RX queue index
2804 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
2805 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
2806 *
2807 * Drivers that implement ndo_rx_flow_steer() should periodically call
2808 * this function for each installed filter and remove the filters for
2809 * which it returns %true.
2810 */
2811bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
2812 u32 flow_id, u16 filter_id)
2813{
2814 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
2815 struct rps_dev_flow_table *flow_table;
2816 struct rps_dev_flow *rflow;
2817 bool expire = true;
2818 int cpu;
2819
2820 rcu_read_lock();
2821 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2822 if (flow_table && flow_id <= flow_table->mask) {
2823 rflow = &flow_table->flows[flow_id];
2824 cpu = ACCESS_ONCE(rflow->cpu);
2825 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
2826 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
2827 rflow->last_qtail) <
2828 (int)(10 * flow_table->mask)))
2829 expire = false;
2830 }
2831 rcu_read_unlock();
2832 return expire;
2833}
2834EXPORT_SYMBOL(rps_may_expire_flow);
2835
2836#endif /* CONFIG_RFS_ACCEL */
2837
Tom Herbert0a9627f2010-03-16 08:03:29 +00002838/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002839static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002840{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002841 struct softnet_data *sd = data;
2842
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002843 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00002844 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002845}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002846
Tom Herbertfec5e652010-04-16 16:01:27 -07002847#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00002848
2849/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002850 * Check if this softnet_data structure is another cpu one
2851 * If yes, queue it to our IPI list and return 1
2852 * If no, return 0
2853 */
2854static int rps_ipi_queued(struct softnet_data *sd)
2855{
2856#ifdef CONFIG_RPS
2857 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2858
2859 if (sd != mysd) {
2860 sd->rps_ipi_next = mysd->rps_ipi_list;
2861 mysd->rps_ipi_list = sd;
2862
2863 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2864 return 1;
2865 }
2866#endif /* CONFIG_RPS */
2867 return 0;
2868}
2869
2870/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00002871 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2872 * queue (may be a remote CPU queue).
2873 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002874static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2875 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002876{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002877 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002878 unsigned long flags;
2879
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002880 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002881
2882 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002883
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002884 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07002885 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2886 if (skb_queue_len(&sd->input_pkt_queue)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002887enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002888 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00002889 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002890 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00002891 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002892 return NET_RX_SUCCESS;
2893 }
2894
Eric Dumazetebda37c22010-05-06 23:51:21 +00002895 /* Schedule NAPI for backlog device
2896 * We can use non atomic operation since we own the queue lock
2897 */
2898 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002899 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002900 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002901 }
2902 goto enqueue;
2903 }
2904
Changli Gaodee42872010-05-02 05:42:16 +00002905 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002906 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002907
Tom Herbert0a9627f2010-03-16 08:03:29 +00002908 local_irq_restore(flags);
2909
Eric Dumazetcaf586e2010-09-30 21:06:55 +00002910 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002911 kfree_skb(skb);
2912 return NET_RX_DROP;
2913}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002914
Linus Torvalds1da177e2005-04-16 15:20:36 -07002915/**
2916 * netif_rx - post buffer to the network code
2917 * @skb: buffer to post
2918 *
2919 * This function receives a packet from a device driver and queues it for
2920 * the upper (protocol) levels to process. It always succeeds. The buffer
2921 * may be dropped during processing for congestion control or by the
2922 * protocol layers.
2923 *
2924 * return values:
2925 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002926 * NET_RX_DROP (packet was dropped)
2927 *
2928 */
2929
2930int netif_rx(struct sk_buff *skb)
2931{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002932 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002933
2934 /* if netpoll wants it, pretend we never saw it */
2935 if (netpoll_rx(skb))
2936 return NET_RX_DROP;
2937
Eric Dumazet588f0332011-11-15 04:12:55 +00002938 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002939
Koki Sanagicf66ba52010-08-23 18:45:02 +09002940 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00002941#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01002942 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002943 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002944 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002945
Changli Gaocece1942010-08-07 20:35:43 -07002946 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002947 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07002948
2949 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002950 if (cpu < 0)
2951 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07002952
2953 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2954
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002955 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07002956 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00002957 } else
2958#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07002959 {
2960 unsigned int qtail;
2961 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
2962 put_cpu();
2963 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002964 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002965}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002966EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002967
2968int netif_rx_ni(struct sk_buff *skb)
2969{
2970 int err;
2971
2972 preempt_disable();
2973 err = netif_rx(skb);
2974 if (local_softirq_pending())
2975 do_softirq();
2976 preempt_enable();
2977
2978 return err;
2979}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002980EXPORT_SYMBOL(netif_rx_ni);
2981
Linus Torvalds1da177e2005-04-16 15:20:36 -07002982static void net_tx_action(struct softirq_action *h)
2983{
2984 struct softnet_data *sd = &__get_cpu_var(softnet_data);
2985
2986 if (sd->completion_queue) {
2987 struct sk_buff *clist;
2988
2989 local_irq_disable();
2990 clist = sd->completion_queue;
2991 sd->completion_queue = NULL;
2992 local_irq_enable();
2993
2994 while (clist) {
2995 struct sk_buff *skb = clist;
2996 clist = clist->next;
2997
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002998 WARN_ON(atomic_read(&skb->users));
Koki Sanagi07dc22e2010-08-23 18:46:12 +09002999 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003000 __kfree_skb(skb);
3001 }
3002 }
3003
3004 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003005 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003006
3007 local_irq_disable();
3008 head = sd->output_queue;
3009 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003010 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003011 local_irq_enable();
3012
3013 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003014 struct Qdisc *q = head;
3015 spinlock_t *root_lock;
3016
Linus Torvalds1da177e2005-04-16 15:20:36 -07003017 head = head->next_sched;
3018
David S. Miller5fb66222008-08-02 20:02:43 -07003019 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003020 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003021 smp_mb__before_clear_bit();
3022 clear_bit(__QDISC_STATE_SCHED,
3023 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003024 qdisc_run(q);
3025 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003026 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003027 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003028 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003029 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003030 } else {
3031 smp_mb__before_clear_bit();
3032 clear_bit(__QDISC_STATE_SCHED,
3033 &q->state);
3034 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003035 }
3036 }
3037 }
3038}
3039
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003040#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3041 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003042/* This hook is defined here for ATM LANE */
3043int (*br_fdb_test_addr_hook)(struct net_device *dev,
3044 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003045EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003046#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003047
Linus Torvalds1da177e2005-04-16 15:20:36 -07003048#ifdef CONFIG_NET_CLS_ACT
3049/* TODO: Maybe we should just force sch_ingress to be compiled in
3050 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3051 * a compare and 2 stores extra right now if we dont have it on
3052 * but have CONFIG_NET_CLS_ACT
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003053 * NOTE: This doesn't stop any functionality; if you dont have
3054 * the ingress scheduler, you just can't add policies on ingress.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003055 *
3056 */
Eric Dumazet24824a02010-10-02 06:11:55 +00003057static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003059 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003060 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07003061 int result = TC_ACT_OK;
3062 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003063
Stephen Hemmingerde384832010-08-01 00:33:23 -07003064 if (unlikely(MAX_RED_LOOP < ttl++)) {
Joe Perchese87cc472012-05-13 21:56:26 +00003065 net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
3066 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07003067 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003068 }
3069
Herbert Xuf697c3e2007-10-14 00:38:47 -07003070 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3071 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3072
David S. Miller83874002008-07-17 00:53:03 -07003073 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07003074 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07003075 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07003076 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3077 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07003078 spin_unlock(qdisc_lock(q));
3079 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07003080
Linus Torvalds1da177e2005-04-16 15:20:36 -07003081 return result;
3082}
Herbert Xuf697c3e2007-10-14 00:38:47 -07003083
3084static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3085 struct packet_type **pt_prev,
3086 int *ret, struct net_device *orig_dev)
3087{
Eric Dumazet24824a02010-10-02 06:11:55 +00003088 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3089
3090 if (!rxq || rxq->qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003091 goto out;
3092
3093 if (*pt_prev) {
3094 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3095 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003096 }
3097
Eric Dumazet24824a02010-10-02 06:11:55 +00003098 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07003099 case TC_ACT_SHOT:
3100 case TC_ACT_STOLEN:
3101 kfree_skb(skb);
3102 return NULL;
3103 }
3104
3105out:
3106 skb->tc_verd = 0;
3107 return skb;
3108}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003109#endif
3110
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003111/**
3112 * netdev_rx_handler_register - register receive handler
3113 * @dev: device to register a handler for
3114 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003115 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003116 *
3117 * Register a receive hander for a device. This handler will then be
3118 * called from __netif_receive_skb. A negative errno code is returned
3119 * on a failure.
3120 *
3121 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003122 *
3123 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003124 */
3125int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003126 rx_handler_func_t *rx_handler,
3127 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003128{
3129 ASSERT_RTNL();
3130
3131 if (dev->rx_handler)
3132 return -EBUSY;
3133
Jiri Pirko93e2c322010-06-10 03:34:59 +00003134 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003135 rcu_assign_pointer(dev->rx_handler, rx_handler);
3136
3137 return 0;
3138}
3139EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3140
3141/**
3142 * netdev_rx_handler_unregister - unregister receive handler
3143 * @dev: device to unregister a handler from
3144 *
3145 * Unregister a receive hander from a device.
3146 *
3147 * The caller must hold the rtnl_mutex.
3148 */
3149void netdev_rx_handler_unregister(struct net_device *dev)
3150{
3151
3152 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003153 RCU_INIT_POINTER(dev->rx_handler, NULL);
3154 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003155}
3156EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3157
Eric Dumazet10f744d2010-03-28 23:07:20 -07003158static int __netif_receive_skb(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003159{
3160 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003161 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003162 struct net_device *orig_dev;
David S. Miller63d8ea72011-02-28 10:48:59 -08003163 struct net_device *null_or_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003164 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003165 int ret = NET_RX_DROP;
Al Viro252e33462006-11-14 20:48:11 -08003166 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003167
Eric Dumazet588f0332011-11-15 04:12:55 +00003168 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003169
Koki Sanagicf66ba52010-08-23 18:45:02 +09003170 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003171
Linus Torvalds1da177e2005-04-16 15:20:36 -07003172 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003173 if (netpoll_receive_skb(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003174 return NET_RX_DROP;
3175
Eric Dumazet8964be42009-11-20 15:35:04 -08003176 if (!skb->skb_iif)
3177 skb->skb_iif = skb->dev->ifindex;
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003178 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003179
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003180 skb_reset_network_header(skb);
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -03003181 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003182 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003183
3184 pt_prev = NULL;
3185
3186 rcu_read_lock();
3187
David S. Miller63d8ea72011-02-28 10:48:59 -08003188another_round:
3189
3190 __this_cpu_inc(softnet_data.processed);
3191
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003192 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3193 skb = vlan_untag(skb);
3194 if (unlikely(!skb))
3195 goto out;
3196 }
3197
Linus Torvalds1da177e2005-04-16 15:20:36 -07003198#ifdef CONFIG_NET_CLS_ACT
3199 if (skb->tc_verd & TC_NCLS) {
3200 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3201 goto ncls;
3202 }
3203#endif
3204
3205 list_for_each_entry_rcu(ptype, &ptype_all, list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003206 if (!ptype->dev || ptype->dev == skb->dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003207 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003208 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003209 pt_prev = ptype;
3210 }
3211 }
3212
3213#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07003214 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3215 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003216 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003217ncls:
3218#endif
3219
Eric Dumazet6a32e4f2011-10-29 06:13:39 +00003220 rx_handler = rcu_dereference(skb->dev->rx_handler);
John Fastabend24257172011-10-10 09:16:41 +00003221 if (vlan_tx_tag_present(skb)) {
3222 if (pt_prev) {
3223 ret = deliver_skb(skb, pt_prev, orig_dev);
3224 pt_prev = NULL;
3225 }
Eric Dumazet6a32e4f2011-10-29 06:13:39 +00003226 if (vlan_do_receive(&skb, !rx_handler))
John Fastabend24257172011-10-10 09:16:41 +00003227 goto another_round;
3228 else if (unlikely(!skb))
3229 goto out;
3230 }
3231
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003232 if (rx_handler) {
3233 if (pt_prev) {
3234 ret = deliver_skb(skb, pt_prev, orig_dev);
3235 pt_prev = NULL;
3236 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003237 switch (rx_handler(&skb)) {
3238 case RX_HANDLER_CONSUMED:
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003239 goto out;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003240 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003241 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003242 case RX_HANDLER_EXACT:
3243 deliver_exact = true;
3244 case RX_HANDLER_PASS:
3245 break;
3246 default:
3247 BUG();
3248 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003249 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003250
David S. Miller63d8ea72011-02-28 10:48:59 -08003251 /* deliver only exact match when indicated */
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003252 null_or_dev = deliver_exact ? skb->dev : NULL;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00003253
Linus Torvalds1da177e2005-04-16 15:20:36 -07003254 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003255 list_for_each_entry_rcu(ptype,
3256 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003257 if (ptype->type == type &&
Jiri Pirkoe3f48d32011-02-28 20:26:31 +00003258 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3259 ptype->dev == orig_dev)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003260 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003261 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003262 pt_prev = ptype;
3263 }
3264 }
3265
3266 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003267 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
3268 ret = -ENOMEM;
3269 else
3270 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003271 } else {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003272 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003273 kfree_skb(skb);
3274 /* Jamal, now you will not able to escape explaining
3275 * me how you were going to use this. :-)
3276 */
3277 ret = NET_RX_DROP;
3278 }
3279
3280out:
3281 rcu_read_unlock();
3282 return ret;
3283}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003284
3285/**
3286 * netif_receive_skb - process receive buffer from network
3287 * @skb: buffer to process
3288 *
3289 * netif_receive_skb() is the main receive data processing function.
3290 * It always succeeds. The buffer may be dropped during processing
3291 * for congestion control or by the protocol layers.
3292 *
3293 * This function may only be called from softirq context and interrupts
3294 * should be enabled.
3295 *
3296 * Return values (usually ignored):
3297 * NET_RX_SUCCESS: no congestion
3298 * NET_RX_DROP: packet was dropped
3299 */
3300int netif_receive_skb(struct sk_buff *skb)
3301{
Eric Dumazet588f0332011-11-15 04:12:55 +00003302 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07003303
Richard Cochranc1f19b52010-07-17 08:49:36 +00003304 if (skb_defer_rx_timestamp(skb))
3305 return NET_RX_SUCCESS;
3306
Eric Dumazetdf334542010-03-24 19:13:54 +00003307#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003308 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07003309 struct rps_dev_flow voidflow, *rflow = &voidflow;
3310 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003311
Eric Dumazet3b098e22010-05-15 23:57:10 -07003312 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003313
Eric Dumazet3b098e22010-05-15 23:57:10 -07003314 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003315
Eric Dumazet3b098e22010-05-15 23:57:10 -07003316 if (cpu >= 0) {
3317 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3318 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00003319 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07003320 }
Eric Dumazetadc93002011-11-17 03:13:26 +00003321 rcu_read_unlock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003322 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003323#endif
Eric Dumazetadc93002011-11-17 03:13:26 +00003324 return __netif_receive_skb(skb);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003325}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003326EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003327
Eric Dumazet88751272010-04-19 05:07:33 +00003328/* Network device is going away, flush any packets still pending
3329 * Called with irqs disabled.
3330 */
Changli Gao152102c2010-03-30 20:16:22 +00003331static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003332{
Changli Gao152102c2010-03-30 20:16:22 +00003333 struct net_device *dev = arg;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003334 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003335 struct sk_buff *skb, *tmp;
3336
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003337 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003338 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003339 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003340 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003341 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003342 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003343 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003344 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003345 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003346
3347 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3348 if (skb->dev == dev) {
3349 __skb_unlink(skb, &sd->process_queue);
3350 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003351 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003352 }
3353 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003354}
3355
Herbert Xud565b0a2008-12-15 23:38:52 -08003356static int napi_gro_complete(struct sk_buff *skb)
3357{
3358 struct packet_type *ptype;
3359 __be16 type = skb->protocol;
3360 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3361 int err = -ENOENT;
3362
Herbert Xufc59f9a2009-04-14 15:11:06 -07003363 if (NAPI_GRO_CB(skb)->count == 1) {
3364 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003365 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003366 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003367
3368 rcu_read_lock();
3369 list_for_each_entry_rcu(ptype, head, list) {
3370 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3371 continue;
3372
3373 err = ptype->gro_complete(skb);
3374 break;
3375 }
3376 rcu_read_unlock();
3377
3378 if (err) {
3379 WARN_ON(&ptype->list == head);
3380 kfree_skb(skb);
3381 return NET_RX_SUCCESS;
3382 }
3383
3384out:
Herbert Xud565b0a2008-12-15 23:38:52 -08003385 return netif_receive_skb(skb);
3386}
3387
Eric Dumazet86cac582010-08-31 18:25:32 +00003388inline void napi_gro_flush(struct napi_struct *napi)
Herbert Xud565b0a2008-12-15 23:38:52 -08003389{
3390 struct sk_buff *skb, *next;
3391
3392 for (skb = napi->gro_list; skb; skb = next) {
3393 next = skb->next;
3394 skb->next = NULL;
3395 napi_gro_complete(skb);
3396 }
3397
Herbert Xu4ae55442009-02-08 18:00:36 +00003398 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003399 napi->gro_list = NULL;
3400}
Eric Dumazet86cac582010-08-31 18:25:32 +00003401EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003402
Ben Hutchings5b252f02009-10-29 07:17:09 +00003403enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08003404{
3405 struct sk_buff **pp = NULL;
3406 struct packet_type *ptype;
3407 __be16 type = skb->protocol;
3408 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
Herbert Xu0da2afd52008-12-26 14:57:42 -08003409 int same_flow;
Herbert Xud565b0a2008-12-15 23:38:52 -08003410 int mac_len;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003411 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003412
Jarek Poplawskice9e76c2010-08-05 01:19:11 +00003413 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
Herbert Xud565b0a2008-12-15 23:38:52 -08003414 goto normal;
3415
David S. Miller21dc3302010-08-23 00:13:46 -07003416 if (skb_is_gso(skb) || skb_has_frag_list(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08003417 goto normal;
3418
Herbert Xud565b0a2008-12-15 23:38:52 -08003419 rcu_read_lock();
3420 list_for_each_entry_rcu(ptype, head, list) {
Herbert Xud565b0a2008-12-15 23:38:52 -08003421 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3422 continue;
3423
Herbert Xu86911732009-01-29 14:19:50 +00003424 skb_set_network_header(skb, skb_gro_offset(skb));
Herbert Xud565b0a2008-12-15 23:38:52 -08003425 mac_len = skb->network_header - skb->mac_header;
3426 skb->mac_len = mac_len;
3427 NAPI_GRO_CB(skb)->same_flow = 0;
3428 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08003429 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003430
Herbert Xud565b0a2008-12-15 23:38:52 -08003431 pp = ptype->gro_receive(&napi->gro_list, skb);
3432 break;
3433 }
3434 rcu_read_unlock();
3435
3436 if (&ptype->list == head)
3437 goto normal;
3438
Herbert Xu0da2afd52008-12-26 14:57:42 -08003439 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003440 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003441
Herbert Xud565b0a2008-12-15 23:38:52 -08003442 if (pp) {
3443 struct sk_buff *nskb = *pp;
3444
3445 *pp = nskb->next;
3446 nskb->next = NULL;
3447 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00003448 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08003449 }
3450
Herbert Xu0da2afd52008-12-26 14:57:42 -08003451 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08003452 goto ok;
3453
Herbert Xu4ae55442009-02-08 18:00:36 +00003454 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08003455 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08003456
Herbert Xu4ae55442009-02-08 18:00:36 +00003457 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08003458 NAPI_GRO_CB(skb)->count = 1;
Herbert Xu86911732009-01-29 14:19:50 +00003459 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003460 skb->next = napi->gro_list;
3461 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003462 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08003463
Herbert Xuad0f9902009-02-01 01:24:55 -08003464pull:
Herbert Xucb189782009-05-26 18:50:31 +00003465 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3466 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3467
3468 BUG_ON(skb->end - skb->tail < grow);
3469
3470 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3471
3472 skb->tail += grow;
3473 skb->data_len -= grow;
3474
3475 skb_shinfo(skb)->frags[0].page_offset += grow;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003476 skb_frag_size_sub(&skb_shinfo(skb)->frags[0], grow);
Herbert Xucb189782009-05-26 18:50:31 +00003477
Eric Dumazet9e903e02011-10-18 21:00:24 +00003478 if (unlikely(!skb_frag_size(&skb_shinfo(skb)->frags[0]))) {
Ian Campbellea2ab692011-08-22 23:44:58 +00003479 skb_frag_unref(skb, 0);
Herbert Xucb189782009-05-26 18:50:31 +00003480 memmove(skb_shinfo(skb)->frags,
3481 skb_shinfo(skb)->frags + 1,
Jarek Poplawskie5093ae2010-08-11 02:02:10 +00003482 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
Herbert Xucb189782009-05-26 18:50:31 +00003483 }
Herbert Xuad0f9902009-02-01 01:24:55 -08003484 }
3485
Herbert Xud565b0a2008-12-15 23:38:52 -08003486ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003487 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003488
3489normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08003490 ret = GRO_NORMAL;
3491 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08003492}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003493EXPORT_SYMBOL(dev_gro_receive);
3494
Eric Dumazet40d08022010-08-26 22:03:08 -07003495static inline gro_result_t
Ben Hutchings5b252f02009-10-29 07:17:09 +00003496__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003497{
3498 struct sk_buff *p;
Eric Dumazet5ca3b72c2012-02-08 08:51:50 +00003499 unsigned int maclen = skb->dev->hard_header_len;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003500
3501 for (p = napi->gro_list; p; p = p->next) {
Eric Dumazet40d08022010-08-26 22:03:08 -07003502 unsigned long diffs;
3503
3504 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
Jesse Gross3701e512010-10-20 13:56:06 +00003505 diffs |= p->vlan_tci ^ skb->vlan_tci;
Eric Dumazet5ca3b72c2012-02-08 08:51:50 +00003506 if (maclen == ETH_HLEN)
3507 diffs |= compare_ether_header(skb_mac_header(p),
3508 skb_gro_mac_header(skb));
3509 else if (!diffs)
3510 diffs = memcmp(skb_mac_header(p),
3511 skb_gro_mac_header(skb),
3512 maclen);
Eric Dumazet40d08022010-08-26 22:03:08 -07003513 NAPI_GRO_CB(p)->same_flow = !diffs;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003514 NAPI_GRO_CB(p)->flush = 0;
3515 }
3516
3517 return dev_gro_receive(napi, skb);
3518}
Herbert Xu5d38a072009-01-04 16:13:40 -08003519
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003520gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08003521{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003522 switch (ret) {
3523 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003524 if (netif_receive_skb(skb))
3525 ret = GRO_DROP;
3526 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003527
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003528 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08003529 kfree_skb(skb);
3530 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003531
Eric Dumazetdaa86542012-04-19 07:07:40 +00003532 case GRO_MERGED_FREE:
Eric Dumazetd7e88832012-04-30 08:10:34 +00003533 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
3534 kmem_cache_free(skbuff_head_cache, skb);
3535 else
3536 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00003537 break;
3538
Ben Hutchings5b252f02009-10-29 07:17:09 +00003539 case GRO_HELD:
3540 case GRO_MERGED:
3541 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003542 }
3543
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003544 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003545}
3546EXPORT_SYMBOL(napi_skb_finish);
3547
Herbert Xu78a478d2009-05-26 18:50:21 +00003548void skb_gro_reset_offset(struct sk_buff *skb)
3549{
3550 NAPI_GRO_CB(skb)->data_offset = 0;
3551 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00003552 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00003553
Herbert Xu78d3fd02009-05-26 18:50:23 +00003554 if (skb->mac_header == skb->tail &&
Ian Campbellea2ab692011-08-22 23:44:58 +00003555 !PageHighMem(skb_frag_page(&skb_shinfo(skb)->frags[0]))) {
Herbert Xu78a478d2009-05-26 18:50:21 +00003556 NAPI_GRO_CB(skb)->frag0 =
Ian Campbellea2ab692011-08-22 23:44:58 +00003557 skb_frag_address(&skb_shinfo(skb)->frags[0]);
Eric Dumazet9e903e02011-10-18 21:00:24 +00003558 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(&skb_shinfo(skb)->frags[0]);
Herbert Xu74895942009-05-26 18:50:27 +00003559 }
Herbert Xu78a478d2009-05-26 18:50:21 +00003560}
3561EXPORT_SYMBOL(skb_gro_reset_offset);
3562
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003563gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003564{
Herbert Xu86911732009-01-29 14:19:50 +00003565 skb_gro_reset_offset(skb);
3566
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003567 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003568}
3569EXPORT_SYMBOL(napi_gro_receive);
3570
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00003571static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003572{
Herbert Xu96e93ea2009-01-06 10:49:34 -08003573 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00003574 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
3575 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00003576 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08003577 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08003578 skb->skb_iif = 0;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003579
3580 napi->skb = skb;
3581}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003582
Herbert Xu76620aa2009-04-16 02:02:07 -07003583struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08003584{
Herbert Xu5d38a072009-01-04 16:13:40 -08003585 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003586
3587 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00003588 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3589 if (skb)
3590 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003591 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003592 return skb;
3593}
Herbert Xu76620aa2009-04-16 02:02:07 -07003594EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003595
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003596gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3597 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003598{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003599 switch (ret) {
3600 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003601 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003602 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003603
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003604 if (ret == GRO_HELD)
3605 skb_gro_pull(skb, -ETH_HLEN);
3606 else if (netif_receive_skb(skb))
3607 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003608 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003609
3610 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003611 case GRO_MERGED_FREE:
3612 napi_reuse_skb(napi, skb);
3613 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003614
3615 case GRO_MERGED:
3616 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003617 }
3618
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003619 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003620}
3621EXPORT_SYMBOL(napi_frags_finish);
3622
Eric Dumazet4adb9c42012-05-18 20:49:06 +00003623static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003624{
Herbert Xu76620aa2009-04-16 02:02:07 -07003625 struct sk_buff *skb = napi->skb;
3626 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00003627 unsigned int hlen;
3628 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07003629
3630 napi->skb = NULL;
3631
3632 skb_reset_mac_header(skb);
3633 skb_gro_reset_offset(skb);
3634
Herbert Xua5b1cf22009-05-26 18:50:28 +00003635 off = skb_gro_offset(skb);
3636 hlen = off + sizeof(*eth);
3637 eth = skb_gro_header_fast(skb, off);
3638 if (skb_gro_header_hard(skb, hlen)) {
3639 eth = skb_gro_header_slow(skb, hlen, off);
3640 if (unlikely(!eth)) {
3641 napi_reuse_skb(napi, skb);
3642 skb = NULL;
3643 goto out;
3644 }
Herbert Xu76620aa2009-04-16 02:02:07 -07003645 }
3646
3647 skb_gro_pull(skb, sizeof(*eth));
3648
3649 /*
3650 * This works because the only protocols we care about don't require
3651 * special handling. We'll fix it up properly at the end.
3652 */
3653 skb->protocol = eth->h_proto;
3654
3655out:
3656 return skb;
3657}
Herbert Xu76620aa2009-04-16 02:02:07 -07003658
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003659gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07003660{
3661 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003662
3663 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003664 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003665
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003666 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08003667}
3668EXPORT_SYMBOL(napi_gro_frags);
3669
Eric Dumazete326bed2010-04-22 00:22:45 -07003670/*
3671 * net_rps_action sends any pending IPI's for rps.
3672 * Note: called with local irq disabled, but exits with local irq enabled.
3673 */
3674static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3675{
3676#ifdef CONFIG_RPS
3677 struct softnet_data *remsd = sd->rps_ipi_list;
3678
3679 if (remsd) {
3680 sd->rps_ipi_list = NULL;
3681
3682 local_irq_enable();
3683
3684 /* Send pending IPI's to kick RPS processing on remote cpus. */
3685 while (remsd) {
3686 struct softnet_data *next = remsd->rps_ipi_next;
3687
3688 if (cpu_online(remsd->cpu))
3689 __smp_call_function_single(remsd->cpu,
3690 &remsd->csd, 0);
3691 remsd = next;
3692 }
3693 } else
3694#endif
3695 local_irq_enable();
3696}
3697
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003698static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003699{
3700 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003701 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003702
Eric Dumazete326bed2010-04-22 00:22:45 -07003703#ifdef CONFIG_RPS
3704 /* Check if we have pending ipi, its better to send them now,
3705 * not waiting net_rx_action() end.
3706 */
3707 if (sd->rps_ipi_list) {
3708 local_irq_disable();
3709 net_rps_action_and_irq_enable(sd);
3710 }
3711#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003712 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07003713 local_irq_disable();
3714 while (work < quota) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003715 struct sk_buff *skb;
Changli Gao6e7676c2010-04-27 15:07:33 -07003716 unsigned int qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003717
Changli Gao6e7676c2010-04-27 15:07:33 -07003718 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07003719 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07003720 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07003721 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00003722 input_queue_head_incr(sd);
3723 if (++work >= quota) {
3724 local_irq_enable();
3725 return work;
3726 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003727 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003728
Changli Gao6e7676c2010-04-27 15:07:33 -07003729 rps_lock(sd);
3730 qlen = skb_queue_len(&sd->input_pkt_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003731 if (qlen)
Changli Gao6e7676c2010-04-27 15:07:33 -07003732 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3733 &sd->process_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003734
Changli Gao6e7676c2010-04-27 15:07:33 -07003735 if (qlen < quota - work) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003736 /*
3737 * Inline a custom version of __napi_complete().
3738 * only current cpu owns and manipulates this napi,
3739 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3740 * we can use a plain write instead of clear_bit(),
3741 * and we dont need an smp_mb() memory barrier.
3742 */
3743 list_del(&napi->poll_list);
3744 napi->state = 0;
3745
Changli Gao6e7676c2010-04-27 15:07:33 -07003746 quota = work + qlen;
3747 }
3748 rps_unlock(sd);
3749 }
3750 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003751
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003752 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003753}
3754
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003755/**
3756 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07003757 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003758 *
3759 * The entry's receive function will be scheduled to run
3760 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08003761void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003762{
3763 unsigned long flags;
3764
3765 local_irq_save(flags);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003766 ____napi_schedule(&__get_cpu_var(softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003767 local_irq_restore(flags);
3768}
3769EXPORT_SYMBOL(__napi_schedule);
3770
Herbert Xud565b0a2008-12-15 23:38:52 -08003771void __napi_complete(struct napi_struct *n)
3772{
3773 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3774 BUG_ON(n->gro_list);
3775
3776 list_del(&n->poll_list);
3777 smp_mb__before_clear_bit();
3778 clear_bit(NAPI_STATE_SCHED, &n->state);
3779}
3780EXPORT_SYMBOL(__napi_complete);
3781
3782void napi_complete(struct napi_struct *n)
3783{
3784 unsigned long flags;
3785
3786 /*
3787 * don't let napi dequeue from the cpu poll list
3788 * just in case its running on a different cpu
3789 */
3790 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3791 return;
3792
3793 napi_gro_flush(n);
3794 local_irq_save(flags);
3795 __napi_complete(n);
3796 local_irq_restore(flags);
3797}
3798EXPORT_SYMBOL(napi_complete);
3799
3800void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3801 int (*poll)(struct napi_struct *, int), int weight)
3802{
3803 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00003804 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003805 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08003806 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003807 napi->poll = poll;
3808 napi->weight = weight;
3809 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08003810 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08003811#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08003812 spin_lock_init(&napi->poll_lock);
3813 napi->poll_owner = -1;
3814#endif
3815 set_bit(NAPI_STATE_SCHED, &napi->state);
3816}
3817EXPORT_SYMBOL(netif_napi_add);
3818
3819void netif_napi_del(struct napi_struct *napi)
3820{
3821 struct sk_buff *skb, *next;
3822
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08003823 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07003824 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08003825
3826 for (skb = napi->gro_list; skb; skb = next) {
3827 next = skb->next;
3828 skb->next = NULL;
3829 kfree_skb(skb);
3830 }
3831
3832 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00003833 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003834}
3835EXPORT_SYMBOL(netif_napi_del);
3836
Linus Torvalds1da177e2005-04-16 15:20:36 -07003837static void net_rx_action(struct softirq_action *h)
3838{
Eric Dumazete326bed2010-04-22 00:22:45 -07003839 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003840 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07003841 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07003842 void *have;
3843
Linus Torvalds1da177e2005-04-16 15:20:36 -07003844 local_irq_disable();
3845
Eric Dumazete326bed2010-04-22 00:22:45 -07003846 while (!list_empty(&sd->poll_list)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003847 struct napi_struct *n;
3848 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003849
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003850 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003851 * Allow this to run for 2 jiffies since which will allow
3852 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003853 */
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003854 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003855 goto softnet_break;
3856
3857 local_irq_enable();
3858
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003859 /* Even though interrupts have been re-enabled, this
3860 * access is safe because interrupts can only add new
3861 * entries to the tail of this list, and only ->poll()
3862 * calls can remove this head entry from the list.
3863 */
Eric Dumazete326bed2010-04-22 00:22:45 -07003864 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003865
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003866 have = netpoll_poll_lock(n);
3867
3868 weight = n->weight;
3869
David S. Miller0a7606c2007-10-29 21:28:47 -07003870 /* This NAPI_STATE_SCHED test is for avoiding a race
3871 * with netpoll's poll_napi(). Only the entity which
3872 * obtains the lock and sees NAPI_STATE_SCHED set will
3873 * actually make the ->poll() call. Therefore we avoid
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003874 * accidentally calling ->poll() when NAPI is not scheduled.
David S. Miller0a7606c2007-10-29 21:28:47 -07003875 */
3876 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00003877 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07003878 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00003879 trace_napi_poll(n);
3880 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003881
3882 WARN_ON_ONCE(work > weight);
3883
3884 budget -= work;
3885
3886 local_irq_disable();
3887
3888 /* Drivers must not modify the NAPI state if they
3889 * consume the entire weight. In such cases this code
3890 * still "owns" the NAPI instance and therefore can
3891 * move the instance around on the list at-will.
3892 */
David S. Millerfed17f32008-01-07 21:00:40 -08003893 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07003894 if (unlikely(napi_disable_pending(n))) {
3895 local_irq_enable();
3896 napi_complete(n);
3897 local_irq_disable();
3898 } else
Eric Dumazete326bed2010-04-22 00:22:45 -07003899 list_move_tail(&n->poll_list, &sd->poll_list);
David S. Millerfed17f32008-01-07 21:00:40 -08003900 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003901
3902 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003903 }
3904out:
Eric Dumazete326bed2010-04-22 00:22:45 -07003905 net_rps_action_and_irq_enable(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003906
Chris Leechdb217332006-06-17 21:24:58 -07003907#ifdef CONFIG_NET_DMA
3908 /*
3909 * There may not be any more sk_buffs coming right now, so push
3910 * any pending DMA copies to hardware
3911 */
Dan Williams2ba05622009-01-06 11:38:14 -07003912 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07003913#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003914
Linus Torvalds1da177e2005-04-16 15:20:36 -07003915 return;
3916
3917softnet_break:
Changli Gaodee42872010-05-02 05:42:16 +00003918 sd->time_squeeze++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003919 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3920 goto out;
3921}
3922
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003923static gifconf_func_t *gifconf_list[NPROTO];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003924
3925/**
3926 * register_gifconf - register a SIOCGIF handler
3927 * @family: Address family
3928 * @gifconf: Function handler
3929 *
3930 * Register protocol dependent address dumping routines. The handler
3931 * that is passed must not be freed or reused until it has been replaced
3932 * by another handler.
3933 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003934int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003935{
3936 if (family >= NPROTO)
3937 return -EINVAL;
3938 gifconf_list[family] = gifconf;
3939 return 0;
3940}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003941EXPORT_SYMBOL(register_gifconf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003942
3943
3944/*
3945 * Map an interface index to its name (SIOCGIFNAME)
3946 */
3947
3948/*
3949 * We need this ioctl for efficient implementation of the
3950 * if_indextoname() function required by the IPv6 API. Without
3951 * it, we would have to search all the interfaces to find a
3952 * match. --pb
3953 */
3954
Eric W. Biederman881d9662007-09-17 11:56:21 -07003955static int dev_ifname(struct net *net, struct ifreq __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003956{
3957 struct net_device *dev;
3958 struct ifreq ifr;
3959
3960 /*
3961 * Fetch the caller's info block.
3962 */
3963
3964 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
3965 return -EFAULT;
3966
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003967 rcu_read_lock();
3968 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003969 if (!dev) {
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003970 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003971 return -ENODEV;
3972 }
3973
3974 strcpy(ifr.ifr_name, dev->name);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003975 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003976
3977 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
3978 return -EFAULT;
3979 return 0;
3980}
3981
3982/*
3983 * Perform a SIOCGIFCONF call. This structure will change
3984 * size eventually, and there is nothing I can do about it.
3985 * Thus we will need a 'compatibility mode'.
3986 */
3987
Eric W. Biederman881d9662007-09-17 11:56:21 -07003988static int dev_ifconf(struct net *net, char __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003989{
3990 struct ifconf ifc;
3991 struct net_device *dev;
3992 char __user *pos;
3993 int len;
3994 int total;
3995 int i;
3996
3997 /*
3998 * Fetch the caller's info block.
3999 */
4000
4001 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
4002 return -EFAULT;
4003
4004 pos = ifc.ifc_buf;
4005 len = ifc.ifc_len;
4006
4007 /*
4008 * Loop over the interfaces, and write an info block for each.
4009 */
4010
4011 total = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004012 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004013 for (i = 0; i < NPROTO; i++) {
4014 if (gifconf_list[i]) {
4015 int done;
4016 if (!pos)
4017 done = gifconf_list[i](dev, NULL, 0);
4018 else
4019 done = gifconf_list[i](dev, pos + total,
4020 len - total);
4021 if (done < 0)
4022 return -EFAULT;
4023 total += done;
4024 }
4025 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004026 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004027
4028 /*
4029 * All done. Write the updated control block back to the caller.
4030 */
4031 ifc.ifc_len = total;
4032
4033 /*
4034 * Both BSD and Solaris return 0 here, so we do too.
4035 */
4036 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
4037}
4038
4039#ifdef CONFIG_PROC_FS
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004040
Eric Dumazet2def16a2012-04-02 22:33:02 +00004041#define BUCKET_SPACE (32 - NETDEV_HASHBITS - 1)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004042
4043#define get_bucket(x) ((x) >> BUCKET_SPACE)
4044#define get_offset(x) ((x) & ((1 << BUCKET_SPACE) - 1))
4045#define set_bucket_offset(b, o) ((b) << BUCKET_SPACE | (o))
4046
Eric Dumazet2def16a2012-04-02 22:33:02 +00004047static inline struct net_device *dev_from_same_bucket(struct seq_file *seq, loff_t *pos)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004048{
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004049 struct net *net = seq_file_net(seq);
4050 struct net_device *dev;
4051 struct hlist_node *p;
4052 struct hlist_head *h;
Eric Dumazet2def16a2012-04-02 22:33:02 +00004053 unsigned int count = 0, offset = get_offset(*pos);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004054
Eric Dumazet2def16a2012-04-02 22:33:02 +00004055 h = &net->dev_name_head[get_bucket(*pos)];
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004056 hlist_for_each_entry_rcu(dev, p, h, name_hlist) {
Eric Dumazet2def16a2012-04-02 22:33:02 +00004057 if (++count == offset)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004058 return dev;
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004059 }
4060
4061 return NULL;
4062}
4063
Eric Dumazet2def16a2012-04-02 22:33:02 +00004064static inline struct net_device *dev_from_bucket(struct seq_file *seq, loff_t *pos)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004065{
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004066 struct net_device *dev;
4067 unsigned int bucket;
4068
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004069 do {
Eric Dumazet2def16a2012-04-02 22:33:02 +00004070 dev = dev_from_same_bucket(seq, pos);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004071 if (dev)
4072 return dev;
4073
Eric Dumazet2def16a2012-04-02 22:33:02 +00004074 bucket = get_bucket(*pos) + 1;
4075 *pos = set_bucket_offset(bucket, 1);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004076 } while (bucket < NETDEV_HASHENTRIES);
4077
4078 return NULL;
4079}
4080
Linus Torvalds1da177e2005-04-16 15:20:36 -07004081/*
4082 * This is invoked by the /proc filesystem handler to display a device
4083 * in detail.
4084 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004085void *dev_seq_start(struct seq_file *seq, loff_t *pos)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004086 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004087{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004088 rcu_read_lock();
Pavel Emelianov7562f872007-05-03 15:13:45 -07004089 if (!*pos)
4090 return SEQ_START_TOKEN;
4091
Eric Dumazet2def16a2012-04-02 22:33:02 +00004092 if (get_bucket(*pos) >= NETDEV_HASHENTRIES)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004093 return NULL;
Pavel Emelianov7562f872007-05-03 15:13:45 -07004094
Eric Dumazet2def16a2012-04-02 22:33:02 +00004095 return dev_from_bucket(seq, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004096}
4097
4098void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4099{
4100 ++*pos;
Eric Dumazet2def16a2012-04-02 22:33:02 +00004101 return dev_from_bucket(seq, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004102}
4103
4104void dev_seq_stop(struct seq_file *seq, void *v)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004105 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004106{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004107 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004108}
4109
4110static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
4111{
Eric Dumazet28172732010-07-07 14:58:56 -07004112 struct rtnl_link_stats64 temp;
4113 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004114
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +00004115 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
4116 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
Rusty Russell5a1b5892007-04-28 21:04:03 -07004117 dev->name, stats->rx_bytes, stats->rx_packets,
4118 stats->rx_errors,
4119 stats->rx_dropped + stats->rx_missed_errors,
4120 stats->rx_fifo_errors,
4121 stats->rx_length_errors + stats->rx_over_errors +
4122 stats->rx_crc_errors + stats->rx_frame_errors,
4123 stats->rx_compressed, stats->multicast,
4124 stats->tx_bytes, stats->tx_packets,
4125 stats->tx_errors, stats->tx_dropped,
4126 stats->tx_fifo_errors, stats->collisions,
4127 stats->tx_carrier_errors +
4128 stats->tx_aborted_errors +
4129 stats->tx_window_errors +
4130 stats->tx_heartbeat_errors,
4131 stats->tx_compressed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004132}
4133
4134/*
4135 * Called from the PROCfs module. This now uses the new arbitrary sized
4136 * /proc/net interface to create /proc/net/dev
4137 */
4138static int dev_seq_show(struct seq_file *seq, void *v)
4139{
4140 if (v == SEQ_START_TOKEN)
4141 seq_puts(seq, "Inter-| Receive "
4142 " | Transmit\n"
4143 " face |bytes packets errs drop fifo frame "
4144 "compressed multicast|bytes packets errs "
4145 "drop fifo colls carrier compressed\n");
4146 else
4147 dev_seq_printf_stats(seq, v);
4148 return 0;
4149}
4150
Changli Gaodee42872010-05-02 05:42:16 +00004151static struct softnet_data *softnet_get_online(loff_t *pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004152{
Changli Gaodee42872010-05-02 05:42:16 +00004153 struct softnet_data *sd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004154
Mike Travis0c0b0ac2008-05-02 16:43:08 -07004155 while (*pos < nr_cpu_ids)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004156 if (cpu_online(*pos)) {
Changli Gaodee42872010-05-02 05:42:16 +00004157 sd = &per_cpu(softnet_data, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004158 break;
4159 } else
4160 ++*pos;
Changli Gaodee42872010-05-02 05:42:16 +00004161 return sd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004162}
4163
4164static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
4165{
4166 return softnet_get_online(pos);
4167}
4168
4169static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4170{
4171 ++*pos;
4172 return softnet_get_online(pos);
4173}
4174
4175static void softnet_seq_stop(struct seq_file *seq, void *v)
4176{
4177}
4178
4179static int softnet_seq_show(struct seq_file *seq, void *v)
4180{
Changli Gaodee42872010-05-02 05:42:16 +00004181 struct softnet_data *sd = v;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004182
Tom Herbert0a9627f2010-03-16 08:03:29 +00004183 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
Changli Gaodee42872010-05-02 05:42:16 +00004184 sd->processed, sd->dropped, sd->time_squeeze, 0,
Stephen Hemmingerc1ebcdb2005-06-23 20:08:59 -07004185 0, 0, 0, 0, /* was fastroute */
Changli Gaodee42872010-05-02 05:42:16 +00004186 sd->cpu_collision, sd->received_rps);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004187 return 0;
4188}
4189
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004190static const struct seq_operations dev_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004191 .start = dev_seq_start,
4192 .next = dev_seq_next,
4193 .stop = dev_seq_stop,
4194 .show = dev_seq_show,
4195};
4196
4197static int dev_seq_open(struct inode *inode, struct file *file)
4198{
Denis V. Luneve372c412007-11-19 22:31:54 -08004199 return seq_open_net(inode, file, &dev_seq_ops,
Eric Dumazet2def16a2012-04-02 22:33:02 +00004200 sizeof(struct seq_net_private));
Anton Blanchard5cac98d2011-11-27 21:14:46 +00004201}
4202
Arjan van de Ven9a321442007-02-12 00:55:35 -08004203static const struct file_operations dev_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004204 .owner = THIS_MODULE,
4205 .open = dev_seq_open,
4206 .read = seq_read,
4207 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004208 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004209};
4210
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004211static const struct seq_operations softnet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004212 .start = softnet_seq_start,
4213 .next = softnet_seq_next,
4214 .stop = softnet_seq_stop,
4215 .show = softnet_seq_show,
4216};
4217
4218static int softnet_seq_open(struct inode *inode, struct file *file)
4219{
4220 return seq_open(file, &softnet_seq_ops);
4221}
4222
Arjan van de Ven9a321442007-02-12 00:55:35 -08004223static const struct file_operations softnet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004224 .owner = THIS_MODULE,
4225 .open = softnet_seq_open,
4226 .read = seq_read,
4227 .llseek = seq_lseek,
4228 .release = seq_release,
4229};
4230
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004231static void *ptype_get_idx(loff_t pos)
4232{
4233 struct packet_type *pt = NULL;
4234 loff_t i = 0;
4235 int t;
4236
4237 list_for_each_entry_rcu(pt, &ptype_all, list) {
4238 if (i == pos)
4239 return pt;
4240 ++i;
4241 }
4242
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004243 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004244 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
4245 if (i == pos)
4246 return pt;
4247 ++i;
4248 }
4249 }
4250 return NULL;
4251}
4252
4253static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004254 __acquires(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004255{
4256 rcu_read_lock();
4257 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
4258}
4259
4260static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4261{
4262 struct packet_type *pt;
4263 struct list_head *nxt;
4264 int hash;
4265
4266 ++*pos;
4267 if (v == SEQ_START_TOKEN)
4268 return ptype_get_idx(0);
4269
4270 pt = v;
4271 nxt = pt->list.next;
4272 if (pt->type == htons(ETH_P_ALL)) {
4273 if (nxt != &ptype_all)
4274 goto found;
4275 hash = 0;
4276 nxt = ptype_base[0].next;
4277 } else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004278 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004279
4280 while (nxt == &ptype_base[hash]) {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004281 if (++hash >= PTYPE_HASH_SIZE)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004282 return NULL;
4283 nxt = ptype_base[hash].next;
4284 }
4285found:
4286 return list_entry(nxt, struct packet_type, list);
4287}
4288
4289static void ptype_seq_stop(struct seq_file *seq, void *v)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004290 __releases(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004291{
4292 rcu_read_unlock();
4293}
4294
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004295static int ptype_seq_show(struct seq_file *seq, void *v)
4296{
4297 struct packet_type *pt = v;
4298
4299 if (v == SEQ_START_TOKEN)
4300 seq_puts(seq, "Type Device Function\n");
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004301 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004302 if (pt->type == htons(ETH_P_ALL))
4303 seq_puts(seq, "ALL ");
4304 else
4305 seq_printf(seq, "%04x", ntohs(pt->type));
4306
Alexey Dobriyan908cd2d2008-11-16 19:50:35 -08004307 seq_printf(seq, " %-8s %pF\n",
4308 pt->dev ? pt->dev->name : "", pt->func);
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004309 }
4310
4311 return 0;
4312}
4313
4314static const struct seq_operations ptype_seq_ops = {
4315 .start = ptype_seq_start,
4316 .next = ptype_seq_next,
4317 .stop = ptype_seq_stop,
4318 .show = ptype_seq_show,
4319};
4320
4321static int ptype_seq_open(struct inode *inode, struct file *file)
4322{
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004323 return seq_open_net(inode, file, &ptype_seq_ops,
4324 sizeof(struct seq_net_private));
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004325}
4326
4327static const struct file_operations ptype_seq_fops = {
4328 .owner = THIS_MODULE,
4329 .open = ptype_seq_open,
4330 .read = seq_read,
4331 .llseek = seq_lseek,
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004332 .release = seq_release_net,
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004333};
4334
4335
Pavel Emelyanov46650792007-10-08 20:38:39 -07004336static int __net_init dev_proc_net_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004337{
4338 int rc = -ENOMEM;
4339
Eric W. Biederman881d9662007-09-17 11:56:21 -07004340 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004341 goto out;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004342 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004343 goto out_dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004344 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004345 goto out_softnet;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004346
Eric W. Biederman881d9662007-09-17 11:56:21 -07004347 if (wext_proc_init(net))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004348 goto out_ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004349 rc = 0;
4350out:
4351 return rc;
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004352out_ptype:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004353 proc_net_remove(net, "ptype");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004354out_softnet:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004355 proc_net_remove(net, "softnet_stat");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004356out_dev:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004357 proc_net_remove(net, "dev");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004358 goto out;
4359}
Eric W. Biederman881d9662007-09-17 11:56:21 -07004360
Pavel Emelyanov46650792007-10-08 20:38:39 -07004361static void __net_exit dev_proc_net_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004362{
4363 wext_proc_exit(net);
4364
4365 proc_net_remove(net, "ptype");
4366 proc_net_remove(net, "softnet_stat");
4367 proc_net_remove(net, "dev");
4368}
4369
Denis V. Lunev022cbae2007-11-13 03:23:50 -08004370static struct pernet_operations __net_initdata dev_proc_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07004371 .init = dev_proc_net_init,
4372 .exit = dev_proc_net_exit,
4373};
4374
4375static int __init dev_proc_init(void)
4376{
4377 return register_pernet_subsys(&dev_proc_ops);
4378}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004379#else
4380#define dev_proc_init() 0
4381#endif /* CONFIG_PROC_FS */
4382
4383
4384/**
Jiri Pirko1765a572011-02-12 06:48:36 +00004385 * netdev_set_master - set up master pointer
Linus Torvalds1da177e2005-04-16 15:20:36 -07004386 * @slave: slave device
4387 * @master: new master device
4388 *
4389 * Changes the master device of the slave. Pass %NULL to break the
4390 * bonding. The caller must hold the RTNL semaphore. On a failure
4391 * a negative errno code is returned. On success the reference counts
Jiri Pirko1765a572011-02-12 06:48:36 +00004392 * are adjusted and the function returns zero.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004393 */
4394int netdev_set_master(struct net_device *slave, struct net_device *master)
4395{
4396 struct net_device *old = slave->master;
4397
4398 ASSERT_RTNL();
4399
4400 if (master) {
4401 if (old)
4402 return -EBUSY;
4403 dev_hold(master);
4404 }
4405
4406 slave->master = master;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004407
Eric Dumazet6df427f2011-05-19 19:37:40 +00004408 if (old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004409 dev_put(old);
Jiri Pirko1765a572011-02-12 06:48:36 +00004410 return 0;
4411}
4412EXPORT_SYMBOL(netdev_set_master);
4413
4414/**
4415 * netdev_set_bond_master - set up bonding master/slave pair
4416 * @slave: slave device
4417 * @master: new master device
4418 *
4419 * Changes the master device of the slave. Pass %NULL to break the
4420 * bonding. The caller must hold the RTNL semaphore. On a failure
4421 * a negative errno code is returned. On success %RTM_NEWLINK is sent
4422 * to the routing socket and the function returns zero.
4423 */
4424int netdev_set_bond_master(struct net_device *slave, struct net_device *master)
4425{
4426 int err;
4427
4428 ASSERT_RTNL();
4429
4430 err = netdev_set_master(slave, master);
4431 if (err)
4432 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004433 if (master)
4434 slave->flags |= IFF_SLAVE;
4435 else
4436 slave->flags &= ~IFF_SLAVE;
4437
4438 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4439 return 0;
4440}
Jiri Pirko1765a572011-02-12 06:48:36 +00004441EXPORT_SYMBOL(netdev_set_bond_master);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004442
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004443static void dev_change_rx_flags(struct net_device *dev, int flags)
4444{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004445 const struct net_device_ops *ops = dev->netdev_ops;
4446
4447 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4448 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004449}
4450
Wang Chendad9b332008-06-18 01:48:28 -07004451static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07004452{
Eric Dumazetb536db92011-11-30 21:42:26 +00004453 unsigned int old_flags = dev->flags;
David Howells8192b0c2008-11-14 10:39:10 +11004454 uid_t uid;
4455 gid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07004456
Patrick McHardy24023452007-07-14 18:51:31 -07004457 ASSERT_RTNL();
4458
Wang Chendad9b332008-06-18 01:48:28 -07004459 dev->flags |= IFF_PROMISC;
4460 dev->promiscuity += inc;
4461 if (dev->promiscuity == 0) {
4462 /*
4463 * Avoid overflow.
4464 * If inc causes overflow, untouch promisc and return error.
4465 */
4466 if (inc < 0)
4467 dev->flags &= ~IFF_PROMISC;
4468 else {
4469 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004470 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
4471 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004472 return -EOVERFLOW;
4473 }
4474 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004475 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004476 pr_info("device %s %s promiscuous mode\n",
4477 dev->name,
4478 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11004479 if (audit_enabled) {
4480 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004481 audit_log(current->audit_context, GFP_ATOMIC,
4482 AUDIT_ANOM_PROMISCUOUS,
4483 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4484 dev->name, (dev->flags & IFF_PROMISC),
4485 (old_flags & IFF_PROMISC),
4486 audit_get_loginuid(current),
David Howells8192b0c2008-11-14 10:39:10 +11004487 uid, gid,
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004488 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11004489 }
Patrick McHardy24023452007-07-14 18:51:31 -07004490
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004491 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07004492 }
Wang Chendad9b332008-06-18 01:48:28 -07004493 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004494}
4495
Linus Torvalds1da177e2005-04-16 15:20:36 -07004496/**
4497 * dev_set_promiscuity - update promiscuity count on a device
4498 * @dev: device
4499 * @inc: modifier
4500 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07004501 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004502 * remains above zero the interface remains promiscuous. Once it hits zero
4503 * the device reverts back to normal filtering operation. A negative inc
4504 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07004505 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004506 */
Wang Chendad9b332008-06-18 01:48:28 -07004507int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004508{
Eric Dumazetb536db92011-11-30 21:42:26 +00004509 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07004510 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004511
Wang Chendad9b332008-06-18 01:48:28 -07004512 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07004513 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07004514 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07004515 if (dev->flags != old_flags)
4516 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07004517 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004518}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004519EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004520
4521/**
4522 * dev_set_allmulti - update allmulti count on a device
4523 * @dev: device
4524 * @inc: modifier
4525 *
4526 * Add or remove reception of all multicast frames to a device. While the
4527 * count in the device remains above zero the interface remains listening
4528 * to all interfaces. Once it hits zero the device reverts back to normal
4529 * filtering operation. A negative @inc value is used to drop the counter
4530 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07004531 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004532 */
4533
Wang Chendad9b332008-06-18 01:48:28 -07004534int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004535{
Eric Dumazetb536db92011-11-30 21:42:26 +00004536 unsigned int old_flags = dev->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004537
Patrick McHardy24023452007-07-14 18:51:31 -07004538 ASSERT_RTNL();
4539
Linus Torvalds1da177e2005-04-16 15:20:36 -07004540 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07004541 dev->allmulti += inc;
4542 if (dev->allmulti == 0) {
4543 /*
4544 * Avoid overflow.
4545 * If inc causes overflow, untouch allmulti and return error.
4546 */
4547 if (inc < 0)
4548 dev->flags &= ~IFF_ALLMULTI;
4549 else {
4550 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004551 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
4552 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004553 return -EOVERFLOW;
4554 }
4555 }
Patrick McHardy24023452007-07-14 18:51:31 -07004556 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004557 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07004558 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07004559 }
Wang Chendad9b332008-06-18 01:48:28 -07004560 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004561}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004562EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07004563
4564/*
4565 * Upload unicast and multicast address lists to device and
4566 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08004567 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07004568 * are present.
4569 */
4570void __dev_set_rx_mode(struct net_device *dev)
4571{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004572 const struct net_device_ops *ops = dev->netdev_ops;
4573
Patrick McHardy4417da62007-06-27 01:28:10 -07004574 /* dev_open will call this function so the list will stay sane. */
4575 if (!(dev->flags&IFF_UP))
4576 return;
4577
4578 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09004579 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07004580
Jiri Pirko01789342011-08-16 06:29:00 +00004581 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004582 /* Unicast addresses changes may only happen under the rtnl,
4583 * therefore calling __dev_set_promiscuity here is safe.
4584 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004585 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004586 __dev_set_promiscuity(dev, 1);
Joe Perches2d348d12011-07-25 16:17:35 -07004587 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004588 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004589 __dev_set_promiscuity(dev, -1);
Joe Perches2d348d12011-07-25 16:17:35 -07004590 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07004591 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004592 }
Jiri Pirko01789342011-08-16 06:29:00 +00004593
4594 if (ops->ndo_set_rx_mode)
4595 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004596}
4597
4598void dev_set_rx_mode(struct net_device *dev)
4599{
David S. Millerb9e40852008-07-15 00:15:08 -07004600 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004601 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07004602 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004603}
4604
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004605/**
4606 * dev_get_flags - get flags reported to userspace
4607 * @dev: device
4608 *
4609 * Get the combination of flag bits exported through APIs to userspace.
4610 */
Eric Dumazet95c96172012-04-15 05:58:06 +00004611unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004612{
Eric Dumazet95c96172012-04-15 05:58:06 +00004613 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004614
4615 flags = (dev->flags & ~(IFF_PROMISC |
4616 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004617 IFF_RUNNING |
4618 IFF_LOWER_UP |
4619 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004620 (dev->gflags & (IFF_PROMISC |
4621 IFF_ALLMULTI));
4622
Stefan Rompfb00055a2006-03-20 17:09:11 -08004623 if (netif_running(dev)) {
4624 if (netif_oper_up(dev))
4625 flags |= IFF_RUNNING;
4626 if (netif_carrier_ok(dev))
4627 flags |= IFF_LOWER_UP;
4628 if (netif_dormant(dev))
4629 flags |= IFF_DORMANT;
4630 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004631
4632 return flags;
4633}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004634EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004635
Patrick McHardybd380812010-02-26 06:34:53 +00004636int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004637{
Eric Dumazetb536db92011-11-30 21:42:26 +00004638 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004639 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004640
Patrick McHardy24023452007-07-14 18:51:31 -07004641 ASSERT_RTNL();
4642
Linus Torvalds1da177e2005-04-16 15:20:36 -07004643 /*
4644 * Set the flags on our device.
4645 */
4646
4647 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4648 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4649 IFF_AUTOMEDIA)) |
4650 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4651 IFF_ALLMULTI));
4652
4653 /*
4654 * Load in the correct multicast list now the flags have changed.
4655 */
4656
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004657 if ((old_flags ^ flags) & IFF_MULTICAST)
4658 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07004659
Patrick McHardy4417da62007-06-27 01:28:10 -07004660 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004661
4662 /*
4663 * Have we downed the interface. We handle IFF_UP ourselves
4664 * according to user attempts to set it, rather than blindly
4665 * setting it.
4666 */
4667
4668 ret = 0;
4669 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00004670 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004671
4672 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07004673 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004674 }
4675
Linus Torvalds1da177e2005-04-16 15:20:36 -07004676 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004677 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4678
Linus Torvalds1da177e2005-04-16 15:20:36 -07004679 dev->gflags ^= IFF_PROMISC;
4680 dev_set_promiscuity(dev, inc);
4681 }
4682
4683 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4684 is important. Some (broken) drivers set IFF_PROMISC, when
4685 IFF_ALLMULTI is requested not asking us and not reporting.
4686 */
4687 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004688 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4689
Linus Torvalds1da177e2005-04-16 15:20:36 -07004690 dev->gflags ^= IFF_ALLMULTI;
4691 dev_set_allmulti(dev, inc);
4692 }
4693
Patrick McHardybd380812010-02-26 06:34:53 +00004694 return ret;
4695}
4696
4697void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4698{
4699 unsigned int changes = dev->flags ^ old_flags;
4700
4701 if (changes & IFF_UP) {
4702 if (dev->flags & IFF_UP)
4703 call_netdevice_notifiers(NETDEV_UP, dev);
4704 else
4705 call_netdevice_notifiers(NETDEV_DOWN, dev);
4706 }
4707
4708 if (dev->flags & IFF_UP &&
4709 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4710 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4711}
4712
4713/**
4714 * dev_change_flags - change device settings
4715 * @dev: device
4716 * @flags: device state flags
4717 *
4718 * Change settings on device based state flags. The flags are
4719 * in the userspace exported format.
4720 */
Eric Dumazetb536db92011-11-30 21:42:26 +00004721int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00004722{
Eric Dumazetb536db92011-11-30 21:42:26 +00004723 int ret;
4724 unsigned int changes, old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004725
4726 ret = __dev_change_flags(dev, flags);
4727 if (ret < 0)
4728 return ret;
4729
4730 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07004731 if (changes)
4732 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004733
Patrick McHardybd380812010-02-26 06:34:53 +00004734 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004735 return ret;
4736}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004737EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004738
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004739/**
4740 * dev_set_mtu - Change maximum transfer unit
4741 * @dev: device
4742 * @new_mtu: new transfer unit
4743 *
4744 * Change the maximum transfer size of the network device.
4745 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004746int dev_set_mtu(struct net_device *dev, int new_mtu)
4747{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004748 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004749 int err;
4750
4751 if (new_mtu == dev->mtu)
4752 return 0;
4753
4754 /* MTU must be positive. */
4755 if (new_mtu < 0)
4756 return -EINVAL;
4757
4758 if (!netif_device_present(dev))
4759 return -ENODEV;
4760
4761 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004762 if (ops->ndo_change_mtu)
4763 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004764 else
4765 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004766
Linus Torvalds1da177e2005-04-16 15:20:36 -07004767 if (!err && dev->flags & IFF_UP)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004768 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004769 return err;
4770}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004771EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004772
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004773/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00004774 * dev_set_group - Change group this device belongs to
4775 * @dev: device
4776 * @new_group: group this device should belong to
4777 */
4778void dev_set_group(struct net_device *dev, int new_group)
4779{
4780 dev->group = new_group;
4781}
4782EXPORT_SYMBOL(dev_set_group);
4783
4784/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004785 * dev_set_mac_address - Change Media Access Control Address
4786 * @dev: device
4787 * @sa: new address
4788 *
4789 * Change the hardware (MAC) address of the device
4790 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004791int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4792{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004793 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004794 int err;
4795
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004796 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004797 return -EOPNOTSUPP;
4798 if (sa->sa_family != dev->type)
4799 return -EINVAL;
4800 if (!netif_device_present(dev))
4801 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004802 err = ops->ndo_set_mac_address(dev, sa);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004803 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004804 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004805 return err;
4806}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004807EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004808
4809/*
Eric Dumazet3710bec2009-11-01 19:42:09 +00004810 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07004811 */
Jeff Garzik14e3e072007-10-08 00:06:32 -07004812static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004813{
4814 int err;
Eric Dumazet3710bec2009-11-01 19:42:09 +00004815 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004816
4817 if (!dev)
4818 return -ENODEV;
4819
4820 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004821 case SIOCGIFFLAGS: /* Get interface flags */
4822 ifr->ifr_flags = (short) dev_get_flags(dev);
4823 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004824
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004825 case SIOCGIFMETRIC: /* Get the metric on the interface
4826 (currently unused) */
4827 ifr->ifr_metric = 0;
4828 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004829
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004830 case SIOCGIFMTU: /* Get the MTU of a device */
4831 ifr->ifr_mtu = dev->mtu;
4832 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004833
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004834 case SIOCGIFHWADDR:
4835 if (!dev->addr_len)
4836 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4837 else
4838 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4839 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4840 ifr->ifr_hwaddr.sa_family = dev->type;
4841 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004842
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004843 case SIOCGIFSLAVE:
4844 err = -EINVAL;
4845 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004846
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004847 case SIOCGIFMAP:
4848 ifr->ifr_map.mem_start = dev->mem_start;
4849 ifr->ifr_map.mem_end = dev->mem_end;
4850 ifr->ifr_map.base_addr = dev->base_addr;
4851 ifr->ifr_map.irq = dev->irq;
4852 ifr->ifr_map.dma = dev->dma;
4853 ifr->ifr_map.port = dev->if_port;
4854 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004855
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004856 case SIOCGIFINDEX:
4857 ifr->ifr_ifindex = dev->ifindex;
4858 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004859
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004860 case SIOCGIFTXQLEN:
4861 ifr->ifr_qlen = dev->tx_queue_len;
4862 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004863
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004864 default:
4865 /* dev_ioctl() should ensure this case
4866 * is never reached
4867 */
4868 WARN_ON(1);
Lifeng Sun41c31f32011-04-27 22:04:51 +00004869 err = -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004870 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004871
4872 }
4873 return err;
4874}
4875
4876/*
4877 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4878 */
4879static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4880{
4881 int err;
4882 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004883 const struct net_device_ops *ops;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004884
4885 if (!dev)
4886 return -ENODEV;
4887
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004888 ops = dev->netdev_ops;
4889
Jeff Garzik14e3e072007-10-08 00:06:32 -07004890 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004891 case SIOCSIFFLAGS: /* Set interface flags */
4892 return dev_change_flags(dev, ifr->ifr_flags);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004893
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004894 case SIOCSIFMETRIC: /* Set the metric on the interface
4895 (currently unused) */
4896 return -EOPNOTSUPP;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004897
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004898 case SIOCSIFMTU: /* Set the MTU of a device */
4899 return dev_set_mtu(dev, ifr->ifr_mtu);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004900
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004901 case SIOCSIFHWADDR:
4902 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004903
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004904 case SIOCSIFHWBROADCAST:
4905 if (ifr->ifr_hwaddr.sa_family != dev->type)
4906 return -EINVAL;
4907 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4908 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4909 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4910 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004911
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004912 case SIOCSIFMAP:
4913 if (ops->ndo_set_config) {
4914 if (!netif_device_present(dev))
4915 return -ENODEV;
4916 return ops->ndo_set_config(dev, &ifr->ifr_map);
4917 }
4918 return -EOPNOTSUPP;
4919
4920 case SIOCADDMULTI:
Jiri Pirkob81693d2011-08-16 06:29:02 +00004921 if (!ops->ndo_set_rx_mode ||
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004922 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4923 return -EINVAL;
4924 if (!netif_device_present(dev))
4925 return -ENODEV;
Jiri Pirko22bedad32010-04-01 21:22:57 +00004926 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004927
4928 case SIOCDELMULTI:
Jiri Pirkob81693d2011-08-16 06:29:02 +00004929 if (!ops->ndo_set_rx_mode ||
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004930 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4931 return -EINVAL;
4932 if (!netif_device_present(dev))
4933 return -ENODEV;
Jiri Pirko22bedad32010-04-01 21:22:57 +00004934 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004935
4936 case SIOCSIFTXQLEN:
4937 if (ifr->ifr_qlen < 0)
4938 return -EINVAL;
4939 dev->tx_queue_len = ifr->ifr_qlen;
4940 return 0;
4941
4942 case SIOCSIFNAME:
4943 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4944 return dev_change_name(dev, ifr->ifr_newname);
4945
Richard Cochran4dc360c2011-10-19 17:00:35 -04004946 case SIOCSHWTSTAMP:
4947 err = net_hwtstamp_validate(ifr);
4948 if (err)
4949 return err;
4950 /* fall through */
4951
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004952 /*
4953 * Unknown or private ioctl
4954 */
4955 default:
4956 if ((cmd >= SIOCDEVPRIVATE &&
4957 cmd <= SIOCDEVPRIVATE + 15) ||
4958 cmd == SIOCBONDENSLAVE ||
4959 cmd == SIOCBONDRELEASE ||
4960 cmd == SIOCBONDSETHWADDR ||
4961 cmd == SIOCBONDSLAVEINFOQUERY ||
4962 cmd == SIOCBONDINFOQUERY ||
4963 cmd == SIOCBONDCHANGEACTIVE ||
4964 cmd == SIOCGMIIPHY ||
4965 cmd == SIOCGMIIREG ||
4966 cmd == SIOCSMIIREG ||
4967 cmd == SIOCBRADDIF ||
4968 cmd == SIOCBRDELIF ||
4969 cmd == SIOCSHWTSTAMP ||
4970 cmd == SIOCWANDEV) {
4971 err = -EOPNOTSUPP;
4972 if (ops->ndo_do_ioctl) {
4973 if (netif_device_present(dev))
4974 err = ops->ndo_do_ioctl(dev, ifr, cmd);
4975 else
4976 err = -ENODEV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004977 }
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004978 } else
4979 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004980
4981 }
4982 return err;
4983}
4984
4985/*
4986 * This function handles all "interface"-type I/O control requests. The actual
4987 * 'doing' part of this is dev_ifsioc above.
4988 */
4989
4990/**
4991 * dev_ioctl - network device ioctl
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004992 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07004993 * @cmd: command to issue
4994 * @arg: pointer to a struct ifreq in user space
4995 *
4996 * Issue ioctl functions to devices. This is normally called by the
4997 * user space syscall interfaces but can sometimes be useful for
4998 * other purposes. The return value is the return from the syscall if
4999 * positive or a negative errno code on error.
5000 */
5001
Eric W. Biederman881d9662007-09-17 11:56:21 -07005002int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005003{
5004 struct ifreq ifr;
5005 int ret;
5006 char *colon;
5007
5008 /* One special case: SIOCGIFCONF takes ifconf argument
5009 and requires shared lock, because it sleeps writing
5010 to user space.
5011 */
5012
5013 if (cmd == SIOCGIFCONF) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005014 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005015 ret = dev_ifconf(net, (char __user *) arg);
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005016 rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005017 return ret;
5018 }
5019 if (cmd == SIOCGIFNAME)
Eric W. Biederman881d9662007-09-17 11:56:21 -07005020 return dev_ifname(net, (struct ifreq __user *)arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005021
5022 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
5023 return -EFAULT;
5024
5025 ifr.ifr_name[IFNAMSIZ-1] = 0;
5026
5027 colon = strchr(ifr.ifr_name, ':');
5028 if (colon)
5029 *colon = 0;
5030
5031 /*
5032 * See which interface the caller is talking about.
5033 */
5034
5035 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005036 /*
5037 * These ioctl calls:
5038 * - can be done by all.
5039 * - atomic and do not require locking.
5040 * - return a value
5041 */
5042 case SIOCGIFFLAGS:
5043 case SIOCGIFMETRIC:
5044 case SIOCGIFMTU:
5045 case SIOCGIFHWADDR:
5046 case SIOCGIFSLAVE:
5047 case SIOCGIFMAP:
5048 case SIOCGIFINDEX:
5049 case SIOCGIFTXQLEN:
5050 dev_load(net, ifr.ifr_name);
Eric Dumazet3710bec2009-11-01 19:42:09 +00005051 rcu_read_lock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005052 ret = dev_ifsioc_locked(net, &ifr, cmd);
Eric Dumazet3710bec2009-11-01 19:42:09 +00005053 rcu_read_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005054 if (!ret) {
5055 if (colon)
5056 *colon = ':';
5057 if (copy_to_user(arg, &ifr,
5058 sizeof(struct ifreq)))
5059 ret = -EFAULT;
5060 }
5061 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005062
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005063 case SIOCETHTOOL:
5064 dev_load(net, ifr.ifr_name);
5065 rtnl_lock();
5066 ret = dev_ethtool(net, &ifr);
5067 rtnl_unlock();
5068 if (!ret) {
5069 if (colon)
5070 *colon = ':';
5071 if (copy_to_user(arg, &ifr,
5072 sizeof(struct ifreq)))
5073 ret = -EFAULT;
5074 }
5075 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005076
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005077 /*
5078 * These ioctl calls:
5079 * - require superuser power.
5080 * - require strict serialization.
5081 * - return a value
5082 */
5083 case SIOCGMIIPHY:
5084 case SIOCGMIIREG:
5085 case SIOCSIFNAME:
5086 if (!capable(CAP_NET_ADMIN))
5087 return -EPERM;
5088 dev_load(net, ifr.ifr_name);
5089 rtnl_lock();
5090 ret = dev_ifsioc(net, &ifr, cmd);
5091 rtnl_unlock();
5092 if (!ret) {
5093 if (colon)
5094 *colon = ':';
5095 if (copy_to_user(arg, &ifr,
5096 sizeof(struct ifreq)))
5097 ret = -EFAULT;
5098 }
5099 return ret;
5100
5101 /*
5102 * These ioctl calls:
5103 * - require superuser power.
5104 * - require strict serialization.
5105 * - do not return a value
5106 */
5107 case SIOCSIFFLAGS:
5108 case SIOCSIFMETRIC:
5109 case SIOCSIFMTU:
5110 case SIOCSIFMAP:
5111 case SIOCSIFHWADDR:
5112 case SIOCSIFSLAVE:
5113 case SIOCADDMULTI:
5114 case SIOCDELMULTI:
5115 case SIOCSIFHWBROADCAST:
5116 case SIOCSIFTXQLEN:
5117 case SIOCSMIIREG:
5118 case SIOCBONDENSLAVE:
5119 case SIOCBONDRELEASE:
5120 case SIOCBONDSETHWADDR:
5121 case SIOCBONDCHANGEACTIVE:
5122 case SIOCBRADDIF:
5123 case SIOCBRDELIF:
5124 case SIOCSHWTSTAMP:
5125 if (!capable(CAP_NET_ADMIN))
5126 return -EPERM;
5127 /* fall through */
5128 case SIOCBONDSLAVEINFOQUERY:
5129 case SIOCBONDINFOQUERY:
5130 dev_load(net, ifr.ifr_name);
5131 rtnl_lock();
5132 ret = dev_ifsioc(net, &ifr, cmd);
5133 rtnl_unlock();
5134 return ret;
5135
5136 case SIOCGIFMEM:
5137 /* Get the per device memory space. We can add this but
5138 * currently do not support it */
5139 case SIOCSIFMEM:
5140 /* Set the per device memory buffer space.
5141 * Not applicable in our case */
5142 case SIOCSIFLINK:
Lifeng Sun41c31f32011-04-27 22:04:51 +00005143 return -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005144
5145 /*
5146 * Unknown or private ioctl.
5147 */
5148 default:
5149 if (cmd == SIOCWANDEV ||
5150 (cmd >= SIOCDEVPRIVATE &&
5151 cmd <= SIOCDEVPRIVATE + 15)) {
Eric W. Biederman881d9662007-09-17 11:56:21 -07005152 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005153 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005154 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005155 rtnl_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005156 if (!ret && copy_to_user(arg, &ifr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005157 sizeof(struct ifreq)))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005158 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005159 return ret;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005160 }
5161 /* Take care of Wireless Extensions */
5162 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
5163 return wext_handle_ioctl(net, &ifr, cmd, arg);
Lifeng Sun41c31f32011-04-27 22:04:51 +00005164 return -ENOTTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005165 }
5166}
5167
5168
5169/**
5170 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005171 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005172 *
5173 * Returns a suitable unique value for a new device interface
5174 * number. The caller must hold the rtnl semaphore or the
5175 * dev_base_lock to be sure it remains unique.
5176 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07005177static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005178{
5179 static int ifindex;
5180 for (;;) {
5181 if (++ifindex <= 0)
5182 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005183 if (!__dev_get_by_index(net, ifindex))
Linus Torvalds1da177e2005-04-16 15:20:36 -07005184 return ifindex;
5185 }
5186}
5187
Linus Torvalds1da177e2005-04-16 15:20:36 -07005188/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08005189static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005190
Stephen Hemminger6f05f622007-03-08 20:46:03 -08005191static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005192{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005193 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005194}
5195
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005196static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005197{
Krishna Kumare93737b2009-12-08 22:26:02 +00005198 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005199
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005200 BUG_ON(dev_boot_phase);
5201 ASSERT_RTNL();
5202
Krishna Kumare93737b2009-12-08 22:26:02 +00005203 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005204 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00005205 * for initialization unwind. Remove those
5206 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005207 */
5208 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005209 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
5210 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005211
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005212 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00005213 list_del(&dev->unreg_list);
5214 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005215 }
Eric Dumazet449f4542011-05-19 12:24:16 +00005216 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005217 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00005218 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005219
Octavian Purdila44345722010-12-13 12:44:07 +00005220 /* If device is running, close it first. */
5221 dev_close_many(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005222
Octavian Purdila44345722010-12-13 12:44:07 +00005223 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005224 /* And unlink it from device chain. */
5225 unlist_netdevice(dev);
5226
5227 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005228 }
5229
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005230 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005231
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005232 list_for_each_entry(dev, head, unreg_list) {
5233 /* Shutdown queueing discipline. */
5234 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005235
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005236
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005237 /* Notify protocols, that we are about to destroy
5238 this device. They should clean all the things.
5239 */
5240 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5241
Patrick McHardya2835762010-02-26 06:34:51 +00005242 if (!dev->rtnl_link_ops ||
5243 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5244 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5245
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005246 /*
5247 * Flush the unicast and multicast chains
5248 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005249 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00005250 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005251
5252 if (dev->netdev_ops->ndo_uninit)
5253 dev->netdev_ops->ndo_uninit(dev);
5254
5255 /* Notifier chain MUST detach us from master device. */
5256 WARN_ON(dev->master);
5257
5258 /* Remove entries from kobject tree */
5259 netdev_unregister_kobject(dev);
5260 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005261
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005262 /* Process any work delayed until the end of the batch */
stephen hemmingere5e26d72010-02-24 14:01:38 +00005263 dev = list_first_entry(head, struct net_device, unreg_list);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005264 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5265
Eric W. Biederman850a5452011-10-13 22:25:23 +00005266 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005267
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005268 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005269 dev_put(dev);
5270}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005271
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005272static void rollback_registered(struct net_device *dev)
5273{
5274 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005275
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005276 list_add(&dev->unreg_list, &single);
5277 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00005278 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005279}
5280
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005281static netdev_features_t netdev_fix_features(struct net_device *dev,
5282 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07005283{
Michał Mirosław57422dc2011-01-22 12:14:12 +00005284 /* Fix illegal checksum combinations */
5285 if ((features & NETIF_F_HW_CSUM) &&
5286 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005287 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005288 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5289 }
5290
Herbert Xub63365a2008-10-23 01:11:29 -07005291 /* Fix illegal SG+CSUM combinations. */
5292 if ((features & NETIF_F_SG) &&
5293 !(features & NETIF_F_ALL_CSUM)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005294 netdev_dbg(dev,
5295 "Dropping NETIF_F_SG since no checksum feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005296 features &= ~NETIF_F_SG;
5297 }
5298
5299 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005300 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005301 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005302 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07005303 }
5304
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00005305 /* TSO ECN requires that TSO is present as well. */
5306 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5307 features &= ~NETIF_F_TSO_ECN;
5308
Michał Mirosław212b5732011-02-15 16:59:16 +00005309 /* Software GSO depends on SG. */
5310 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005311 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00005312 features &= ~NETIF_F_GSO;
5313 }
5314
Michał Mirosławacd11302011-01-24 15:45:15 -08005315 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07005316 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00005317 /* maybe split UFO into V4 and V6? */
5318 if (!((features & NETIF_F_GEN_CSUM) ||
5319 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5320 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005321 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005322 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005323 features &= ~NETIF_F_UFO;
5324 }
5325
5326 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005327 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005328 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005329 features &= ~NETIF_F_UFO;
5330 }
5331 }
5332
5333 return features;
5334}
Herbert Xub63365a2008-10-23 01:11:29 -07005335
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005336int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00005337{
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005338 netdev_features_t features;
Michał Mirosław5455c692011-02-15 16:59:17 +00005339 int err = 0;
5340
Michał Mirosław87267482011-04-12 09:56:38 +00005341 ASSERT_RTNL();
5342
Michał Mirosław5455c692011-02-15 16:59:17 +00005343 features = netdev_get_wanted_features(dev);
5344
5345 if (dev->netdev_ops->ndo_fix_features)
5346 features = dev->netdev_ops->ndo_fix_features(dev, features);
5347
5348 /* driver might be less strict about feature dependencies */
5349 features = netdev_fix_features(dev, features);
5350
5351 if (dev->features == features)
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005352 return 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00005353
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005354 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
5355 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00005356
5357 if (dev->netdev_ops->ndo_set_features)
5358 err = dev->netdev_ops->ndo_set_features(dev, features);
5359
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005360 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00005361 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005362 "set_features() failed (%d); wanted %pNF, left %pNF\n",
5363 err, &features, &dev->features);
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005364 return -1;
5365 }
5366
5367 if (!err)
5368 dev->features = features;
5369
5370 return 1;
5371}
5372
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005373/**
5374 * netdev_update_features - recalculate device features
5375 * @dev: the device to check
5376 *
5377 * Recalculate dev->features set and send notifications if it
5378 * has changed. Should be called after driver or hardware dependent
5379 * conditions might have changed that influence the features.
5380 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005381void netdev_update_features(struct net_device *dev)
5382{
5383 if (__netdev_update_features(dev))
5384 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00005385}
5386EXPORT_SYMBOL(netdev_update_features);
5387
Linus Torvalds1da177e2005-04-16 15:20:36 -07005388/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005389 * netdev_change_features - recalculate device features
5390 * @dev: the device to check
5391 *
5392 * Recalculate dev->features set and send notifications even
5393 * if they have not changed. Should be called instead of
5394 * netdev_update_features() if also dev->vlan_features might
5395 * have changed to allow the changes to be propagated to stacked
5396 * VLAN devices.
5397 */
5398void netdev_change_features(struct net_device *dev)
5399{
5400 __netdev_update_features(dev);
5401 netdev_features_change(dev);
5402}
5403EXPORT_SYMBOL(netdev_change_features);
5404
5405/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005406 * netif_stacked_transfer_operstate - transfer operstate
5407 * @rootdev: the root or lower level device to transfer state from
5408 * @dev: the device to transfer operstate to
5409 *
5410 * Transfer operational state from root to device. This is normally
5411 * called when a stacking relationship exists between the root
5412 * device and the device(a leaf device).
5413 */
5414void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5415 struct net_device *dev)
5416{
5417 if (rootdev->operstate == IF_OPER_DORMANT)
5418 netif_dormant_on(dev);
5419 else
5420 netif_dormant_off(dev);
5421
5422 if (netif_carrier_ok(rootdev)) {
5423 if (!netif_carrier_ok(dev))
5424 netif_carrier_on(dev);
5425 } else {
5426 if (netif_carrier_ok(dev))
5427 netif_carrier_off(dev);
5428 }
5429}
5430EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5431
Tom Herbertbf264142010-11-26 08:36:09 +00005432#ifdef CONFIG_RPS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005433static int netif_alloc_rx_queues(struct net_device *dev)
5434{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005435 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00005436 struct netdev_rx_queue *rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005437
Tom Herbertbd25fa72010-10-18 18:00:16 +00005438 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005439
Tom Herbertbd25fa72010-10-18 18:00:16 +00005440 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5441 if (!rx) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005442 pr_err("netdev: Unable to allocate %u rx queues\n", count);
Tom Herbertbd25fa72010-10-18 18:00:16 +00005443 return -ENOMEM;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005444 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00005445 dev->_rx = rx;
5446
Tom Herbertbd25fa72010-10-18 18:00:16 +00005447 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00005448 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005449 return 0;
5450}
Tom Herbertbf264142010-11-26 08:36:09 +00005451#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005452
Changli Gaoaa942102010-12-04 02:31:41 +00005453static void netdev_init_one_queue(struct net_device *dev,
5454 struct netdev_queue *queue, void *_unused)
5455{
5456 /* Initialize queue lock */
5457 spin_lock_init(&queue->_xmit_lock);
5458 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5459 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00005460 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00005461 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00005462#ifdef CONFIG_BQL
5463 dql_init(&queue->dql, HZ);
5464#endif
Changli Gaoaa942102010-12-04 02:31:41 +00005465}
5466
Tom Herberte6484932010-10-18 18:04:39 +00005467static int netif_alloc_netdev_queues(struct net_device *dev)
5468{
5469 unsigned int count = dev->num_tx_queues;
5470 struct netdev_queue *tx;
5471
5472 BUG_ON(count < 1);
5473
5474 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5475 if (!tx) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005476 pr_err("netdev: Unable to allocate %u tx queues\n", count);
Tom Herberte6484932010-10-18 18:04:39 +00005477 return -ENOMEM;
5478 }
5479 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00005480
Tom Herberte6484932010-10-18 18:04:39 +00005481 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5482 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00005483
5484 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00005485}
5486
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005487/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005488 * register_netdevice - register a network device
5489 * @dev: device to register
5490 *
5491 * Take a completed network device structure and add it to the kernel
5492 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5493 * chain. 0 is returned on success. A negative errno code is returned
5494 * on a failure to set up the device, or if the name is a duplicate.
5495 *
5496 * Callers must hold the rtnl semaphore. You may want
5497 * register_netdev() instead of this.
5498 *
5499 * BUGS:
5500 * The locking appears insufficient to guarantee two parallel registers
5501 * will not get the same name.
5502 */
5503
5504int register_netdevice(struct net_device *dev)
5505{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005506 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005507 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005508
5509 BUG_ON(dev_boot_phase);
5510 ASSERT_RTNL();
5511
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005512 might_sleep();
5513
Linus Torvalds1da177e2005-04-16 15:20:36 -07005514 /* When net_device's are persistent, this will be fatal. */
5515 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005516 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005517
David S. Millerf1f28aa2008-07-15 00:08:33 -07005518 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005519 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005520
Linus Torvalds1da177e2005-04-16 15:20:36 -07005521 dev->iflink = -1;
5522
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00005523 ret = dev_get_valid_name(dev, dev->name);
5524 if (ret < 0)
5525 goto out;
5526
Linus Torvalds1da177e2005-04-16 15:20:36 -07005527 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005528 if (dev->netdev_ops->ndo_init) {
5529 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005530 if (ret) {
5531 if (ret > 0)
5532 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005533 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005534 }
5535 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005536
Eric W. Biederman881d9662007-09-17 11:56:21 -07005537 dev->ifindex = dev_new_index(net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005538 if (dev->iflink == -1)
5539 dev->iflink = dev->ifindex;
5540
Michał Mirosław5455c692011-02-15 16:59:17 +00005541 /* Transfer changeable features to wanted_features and enable
5542 * software offloads (GSO and GRO).
5543 */
5544 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00005545 dev->features |= NETIF_F_SOFT_FEATURES;
5546 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005547
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005548 /* Turn on no cache copy if HW is doing checksum */
Michał Mirosław34324dc2011-11-15 15:29:55 +00005549 if (!(dev->flags & IFF_LOOPBACK)) {
5550 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5551 if (dev->features & NETIF_F_ALL_CSUM) {
5552 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5553 dev->features |= NETIF_F_NOCACHE_COPY;
5554 }
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005555 }
5556
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005557 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00005558 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005559 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00005560
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005561 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5562 ret = notifier_to_errno(ret);
5563 if (ret)
5564 goto err_uninit;
5565
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005566 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005567 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005568 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005569 dev->reg_state = NETREG_REGISTERED;
5570
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005571 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00005572
Linus Torvalds1da177e2005-04-16 15:20:36 -07005573 /*
5574 * Default initial state at registry is that the
5575 * device is present.
5576 */
5577
5578 set_bit(__LINK_STATE_PRESENT, &dev->state);
5579
Linus Torvalds1da177e2005-04-16 15:20:36 -07005580 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005581 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005582 list_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005583
5584 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005585 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005586 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005587 if (ret) {
5588 rollback_registered(dev);
5589 dev->reg_state = NETREG_UNREGISTERED;
5590 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005591 /*
5592 * Prevent userspace races by waiting until the network
5593 * device is fully setup before sending notifications.
5594 */
Patrick McHardya2835762010-02-26 06:34:51 +00005595 if (!dev->rtnl_link_ops ||
5596 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5597 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005598
5599out:
5600 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005601
5602err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005603 if (dev->netdev_ops->ndo_uninit)
5604 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005605 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005606}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005607EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005608
5609/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005610 * init_dummy_netdev - init a dummy network device for NAPI
5611 * @dev: device to init
5612 *
5613 * This takes a network device structure and initialize the minimum
5614 * amount of fields so it can be used to schedule NAPI polls without
5615 * registering a full blown interface. This is to be used by drivers
5616 * that need to tie several hardware interfaces to a single NAPI
5617 * poll scheduler due to HW limitations.
5618 */
5619int init_dummy_netdev(struct net_device *dev)
5620{
5621 /* Clear everything. Note we don't initialize spinlocks
5622 * are they aren't supposed to be taken by any of the
5623 * NAPI code and this dummy netdev is supposed to be
5624 * only ever used for NAPI polls
5625 */
5626 memset(dev, 0, sizeof(struct net_device));
5627
5628 /* make sure we BUG if trying to hit standard
5629 * register/unregister code path
5630 */
5631 dev->reg_state = NETREG_DUMMY;
5632
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005633 /* NAPI wants this */
5634 INIT_LIST_HEAD(&dev->napi_list);
5635
5636 /* a dummy interface is started by default */
5637 set_bit(__LINK_STATE_PRESENT, &dev->state);
5638 set_bit(__LINK_STATE_START, &dev->state);
5639
Eric Dumazet29b44332010-10-11 10:22:12 +00005640 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5641 * because users of this 'device' dont need to change
5642 * its refcount.
5643 */
5644
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005645 return 0;
5646}
5647EXPORT_SYMBOL_GPL(init_dummy_netdev);
5648
5649
5650/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005651 * register_netdev - register a network device
5652 * @dev: device to register
5653 *
5654 * Take a completed network device structure and add it to the kernel
5655 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5656 * chain. 0 is returned on success. A negative errno code is returned
5657 * on a failure to set up the device, or if the name is a duplicate.
5658 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005659 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005660 * and expands the device name if you passed a format string to
5661 * alloc_netdev.
5662 */
5663int register_netdev(struct net_device *dev)
5664{
5665 int err;
5666
5667 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005668 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005669 rtnl_unlock();
5670 return err;
5671}
5672EXPORT_SYMBOL(register_netdev);
5673
Eric Dumazet29b44332010-10-11 10:22:12 +00005674int netdev_refcnt_read(const struct net_device *dev)
5675{
5676 int i, refcnt = 0;
5677
5678 for_each_possible_cpu(i)
5679 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5680 return refcnt;
5681}
5682EXPORT_SYMBOL(netdev_refcnt_read);
5683
Ben Hutchings2c530402012-07-10 10:55:09 +00005684/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005685 * netdev_wait_allrefs - wait until all references are gone.
5686 *
5687 * This is called when unregistering network devices.
5688 *
5689 * Any protocol or device that holds a reference should register
5690 * for netdevice notification, and cleanup and put back the
5691 * reference if they receive an UNREGISTER event.
5692 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005693 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005694 */
5695static void netdev_wait_allrefs(struct net_device *dev)
5696{
5697 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00005698 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005699
Eric Dumazete014deb2009-11-17 05:59:21 +00005700 linkwatch_forget_dev(dev);
5701
Linus Torvalds1da177e2005-04-16 15:20:36 -07005702 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00005703 refcnt = netdev_refcnt_read(dev);
5704
5705 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005706 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005707 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005708
5709 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005710 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005711 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
Octavian Purdila395264d2009-11-16 13:49:35 +00005712 * should have already handle it the first time */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005713
5714 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5715 &dev->state)) {
5716 /* We must not have linkwatch events
5717 * pending on unregister. If this
5718 * happens, we simply run the queue
5719 * unscheduled, resulting in a noop
5720 * for this device.
5721 */
5722 linkwatch_run_queue();
5723 }
5724
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005725 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005726
5727 rebroadcast_time = jiffies;
5728 }
5729
5730 msleep(250);
5731
Eric Dumazet29b44332010-10-11 10:22:12 +00005732 refcnt = netdev_refcnt_read(dev);
5733
Linus Torvalds1da177e2005-04-16 15:20:36 -07005734 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005735 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
5736 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005737 warning_time = jiffies;
5738 }
5739 }
5740}
5741
5742/* The sequence is:
5743 *
5744 * rtnl_lock();
5745 * ...
5746 * register_netdevice(x1);
5747 * register_netdevice(x2);
5748 * ...
5749 * unregister_netdevice(y1);
5750 * unregister_netdevice(y2);
5751 * ...
5752 * rtnl_unlock();
5753 * free_netdev(y1);
5754 * free_netdev(y2);
5755 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005756 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005757 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005758 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005759 * without deadlocking with linkwatch via keventd.
5760 * 2) Since we run with the RTNL semaphore not held, we can sleep
5761 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005762 *
5763 * We must not return until all unregister events added during
5764 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005765 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005766void netdev_run_todo(void)
5767{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005768 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005769
Linus Torvalds1da177e2005-04-16 15:20:36 -07005770 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005771 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07005772
5773 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005774
Eric W. Biederman850a5452011-10-13 22:25:23 +00005775 /* Wait for rcu callbacks to finish before attempting to drain
5776 * the device list. This usually avoids a 250ms wait.
5777 */
5778 if (!list_empty(&list))
5779 rcu_barrier();
5780
Linus Torvalds1da177e2005-04-16 15:20:36 -07005781 while (!list_empty(&list)) {
5782 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00005783 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005784 list_del(&dev->todo_list);
5785
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005786 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005787 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07005788 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005789 dump_stack();
5790 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005791 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005792
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005793 dev->reg_state = NETREG_UNREGISTERED;
5794
Changli Gao152102c2010-03-30 20:16:22 +00005795 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005796
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005797 netdev_wait_allrefs(dev);
5798
5799 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00005800 BUG_ON(netdev_refcnt_read(dev));
Eric Dumazet33d480c2011-08-11 19:30:52 +00005801 WARN_ON(rcu_access_pointer(dev->ip_ptr));
5802 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07005803 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005804
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005805 if (dev->destructor)
5806 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07005807
5808 /* Free network device */
5809 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005810 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005811}
5812
Ben Hutchings3cfde792010-07-09 09:11:52 +00005813/* Convert net_device_stats to rtnl_link_stats64. They have the same
5814 * fields in the same order, with only the type differing.
5815 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005816void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5817 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00005818{
5819#if BITS_PER_LONG == 64
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005820 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5821 memcpy(stats64, netdev_stats, sizeof(*stats64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00005822#else
5823 size_t i, n = sizeof(*stats64) / sizeof(u64);
5824 const unsigned long *src = (const unsigned long *)netdev_stats;
5825 u64 *dst = (u64 *)stats64;
5826
5827 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5828 sizeof(*stats64) / sizeof(u64));
5829 for (i = 0; i < n; i++)
5830 dst[i] = src[i];
5831#endif
5832}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005833EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00005834
Eric Dumazetd83345a2009-11-16 03:36:51 +00005835/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005836 * dev_get_stats - get network device statistics
5837 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07005838 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005839 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00005840 * Get network statistics from device. Return @storage.
5841 * The device driver may provide its own method by setting
5842 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5843 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005844 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00005845struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5846 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00005847{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005848 const struct net_device_ops *ops = dev->netdev_ops;
5849
Eric Dumazet28172732010-07-07 14:58:56 -07005850 if (ops->ndo_get_stats64) {
5851 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005852 ops->ndo_get_stats64(dev, storage);
5853 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00005854 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005855 } else {
5856 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07005857 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005858 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07005859 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07005860}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005861EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07005862
Eric Dumazet24824a02010-10-02 06:11:55 +00005863struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07005864{
Eric Dumazet24824a02010-10-02 06:11:55 +00005865 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07005866
Eric Dumazet24824a02010-10-02 06:11:55 +00005867#ifdef CONFIG_NET_CLS_ACT
5868 if (queue)
5869 return queue;
5870 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5871 if (!queue)
5872 return NULL;
5873 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet24824a02010-10-02 06:11:55 +00005874 queue->qdisc = &noop_qdisc;
5875 queue->qdisc_sleeping = &noop_qdisc;
5876 rcu_assign_pointer(dev->ingress_queue, queue);
5877#endif
5878 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07005879}
5880
Linus Torvalds1da177e2005-04-16 15:20:36 -07005881/**
Tom Herbert36909ea2011-01-09 19:36:31 +00005882 * alloc_netdev_mqs - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005883 * @sizeof_priv: size of private data to allocate space for
5884 * @name: device name format string
5885 * @setup: callback to initialize device
Tom Herbert36909ea2011-01-09 19:36:31 +00005886 * @txqs: the number of TX subqueues to allocate
5887 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07005888 *
5889 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005890 * and performs basic initialization. Also allocates subquue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00005891 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005892 */
Tom Herbert36909ea2011-01-09 19:36:31 +00005893struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5894 void (*setup)(struct net_device *),
5895 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005896{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005897 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07005898 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005899 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005900
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005901 BUG_ON(strlen(name) >= sizeof(dev->name));
5902
Tom Herbert36909ea2011-01-09 19:36:31 +00005903 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005904 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00005905 return NULL;
5906 }
5907
Tom Herbert36909ea2011-01-09 19:36:31 +00005908#ifdef CONFIG_RPS
5909 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005910 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00005911 return NULL;
5912 }
5913#endif
5914
David S. Millerfd2ea0a2008-07-17 01:56:23 -07005915 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005916 if (sizeof_priv) {
5917 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005918 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005919 alloc_size += sizeof_priv;
5920 }
5921 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005922 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005923
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07005924 p = kzalloc(alloc_size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005925 if (!p) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005926 pr_err("alloc_netdev: Unable to allocate device\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005927 return NULL;
5928 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005929
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005930 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005931 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005932
Eric Dumazet29b44332010-10-11 10:22:12 +00005933 dev->pcpu_refcnt = alloc_percpu(int);
5934 if (!dev->pcpu_refcnt)
Tom Herberte6484932010-10-18 18:04:39 +00005935 goto free_p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005936
Linus Torvalds1da177e2005-04-16 15:20:36 -07005937 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00005938 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005939
Jiri Pirko22bedad32010-04-01 21:22:57 +00005940 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005941 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00005942
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09005943 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005944
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07005945 dev->gso_max_size = GSO_MAX_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005946
Herbert Xud565b0a2008-12-15 23:38:52 -08005947 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005948 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00005949 INIT_LIST_HEAD(&dev->link_watch_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07005950 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005951 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08005952
5953 dev->num_tx_queues = txqs;
5954 dev->real_num_tx_queues = txqs;
5955 if (netif_alloc_netdev_queues(dev))
5956 goto free_all;
5957
5958#ifdef CONFIG_RPS
5959 dev->num_rx_queues = rxqs;
5960 dev->real_num_rx_queues = rxqs;
5961 if (netif_alloc_rx_queues(dev))
5962 goto free_all;
5963#endif
5964
Linus Torvalds1da177e2005-04-16 15:20:36 -07005965 strcpy(dev->name, name);
Vlad Dogarucbda10f2011-01-13 23:38:30 +00005966 dev->group = INIT_NETDEV_GROUP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005967 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005968
David S. Miller8d3bdbd2011-02-08 15:02:50 -08005969free_all:
5970 free_netdev(dev);
5971 return NULL;
5972
Eric Dumazet29b44332010-10-11 10:22:12 +00005973free_pcpu:
5974 free_percpu(dev->pcpu_refcnt);
Tom Herberted9af2e2010-11-09 10:47:30 +00005975 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00005976#ifdef CONFIG_RPS
5977 kfree(dev->_rx);
5978#endif
5979
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005980free_p:
5981 kfree(p);
5982 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005983}
Tom Herbert36909ea2011-01-09 19:36:31 +00005984EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005985
5986/**
5987 * free_netdev - free network device
5988 * @dev: device
5989 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005990 * This function does the last stage of destroying an allocated device
5991 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005992 * If this is the last reference then it will be freed.
5993 */
5994void free_netdev(struct net_device *dev)
5995{
Herbert Xud565b0a2008-12-15 23:38:52 -08005996 struct napi_struct *p, *n;
5997
Denis V. Lunevf3005d72008-04-16 02:02:18 -07005998 release_net(dev_net(dev));
5999
David S. Millere8a04642008-07-17 00:34:19 -07006000 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00006001#ifdef CONFIG_RPS
6002 kfree(dev->_rx);
6003#endif
David S. Millere8a04642008-07-17 00:34:19 -07006004
Eric Dumazet33d480c2011-08-11 19:30:52 +00006005 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00006006
Jiri Pirkof001fde2009-05-05 02:48:28 +00006007 /* Flush device addresses */
6008 dev_addr_flush(dev);
6009
Herbert Xud565b0a2008-12-15 23:38:52 -08006010 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
6011 netif_napi_del(p);
6012
Eric Dumazet29b44332010-10-11 10:22:12 +00006013 free_percpu(dev->pcpu_refcnt);
6014 dev->pcpu_refcnt = NULL;
6015
Stephen Hemminger3041a062006-05-26 13:25:24 -07006016 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006017 if (dev->reg_state == NETREG_UNINITIALIZED) {
6018 kfree((char *)dev - dev->padded);
6019 return;
6020 }
6021
6022 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
6023 dev->reg_state = NETREG_RELEASED;
6024
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07006025 /* will free via device release */
6026 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006027}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006028EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006029
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006030/**
6031 * synchronize_net - Synchronize with packet receive processing
6032 *
6033 * Wait for packets currently being received to be done.
6034 * Does not block later packets from starting.
6035 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006036void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006037{
6038 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00006039 if (rtnl_is_locked())
6040 synchronize_rcu_expedited();
6041 else
6042 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006043}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006044EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006045
6046/**
Eric Dumazet44a08732009-10-27 07:03:04 +00006047 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07006048 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00006049 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08006050 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07006051 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006052 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00006053 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006054 *
6055 * Callers must hold the rtnl semaphore. You may want
6056 * unregister_netdev() instead of this.
6057 */
6058
Eric Dumazet44a08732009-10-27 07:03:04 +00006059void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006060{
Herbert Xua6620712007-12-12 19:21:56 -08006061 ASSERT_RTNL();
6062
Eric Dumazet44a08732009-10-27 07:03:04 +00006063 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006064 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00006065 } else {
6066 rollback_registered(dev);
6067 /* Finish processing unregister after unlock */
6068 net_set_todo(dev);
6069 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006070}
Eric Dumazet44a08732009-10-27 07:03:04 +00006071EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006072
6073/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006074 * unregister_netdevice_many - unregister many devices
6075 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006076 */
6077void unregister_netdevice_many(struct list_head *head)
6078{
6079 struct net_device *dev;
6080
6081 if (!list_empty(head)) {
6082 rollback_registered_many(head);
6083 list_for_each_entry(dev, head, unreg_list)
6084 net_set_todo(dev);
6085 }
6086}
Eric Dumazet63c80992009-10-27 07:06:49 +00006087EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006088
6089/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006090 * unregister_netdev - remove device from the kernel
6091 * @dev: device
6092 *
6093 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006094 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006095 *
6096 * This is just a wrapper for unregister_netdevice that takes
6097 * the rtnl semaphore. In general you want to use this and not
6098 * unregister_netdevice.
6099 */
6100void unregister_netdev(struct net_device *dev)
6101{
6102 rtnl_lock();
6103 unregister_netdevice(dev);
6104 rtnl_unlock();
6105}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006106EXPORT_SYMBOL(unregister_netdev);
6107
Eric W. Biedermance286d32007-09-12 13:53:49 +02006108/**
6109 * dev_change_net_namespace - move device to different nethost namespace
6110 * @dev: device
6111 * @net: network namespace
6112 * @pat: If not NULL name pattern to try if the current device name
6113 * is already taken in the destination network namespace.
6114 *
6115 * This function shuts down a device interface and moves it
6116 * to a new network namespace. On success 0 is returned, on
6117 * a failure a netagive errno code is returned.
6118 *
6119 * Callers must hold the rtnl semaphore.
6120 */
6121
6122int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
6123{
Eric W. Biedermance286d32007-09-12 13:53:49 +02006124 int err;
6125
6126 ASSERT_RTNL();
6127
6128 /* Don't allow namespace local devices to be moved. */
6129 err = -EINVAL;
6130 if (dev->features & NETIF_F_NETNS_LOCAL)
6131 goto out;
6132
6133 /* Ensure the device has been registrered */
6134 err = -EINVAL;
6135 if (dev->reg_state != NETREG_REGISTERED)
6136 goto out;
6137
6138 /* Get out if there is nothing todo */
6139 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09006140 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02006141 goto out;
6142
6143 /* Pick the destination device name, and ensure
6144 * we can use it in the destination network namespace.
6145 */
6146 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00006147 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006148 /* We get here if we can't use the current device name */
6149 if (!pat)
6150 goto out;
Jiri Pirko1c5cae82011-04-30 01:21:32 +00006151 if (dev_get_valid_name(dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006152 goto out;
6153 }
6154
6155 /*
6156 * And now a mini version of register_netdevice unregister_netdevice.
6157 */
6158
6159 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07006160 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006161
6162 /* And unlink it from device chain */
6163 err = -ENODEV;
6164 unlist_netdevice(dev);
6165
6166 synchronize_net();
6167
6168 /* Shutdown queueing discipline. */
6169 dev_shutdown(dev);
6170
6171 /* Notify protocols, that we are about to destroy
6172 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00006173
6174 Note that dev->reg_state stays at NETREG_REGISTERED.
6175 This is wanted because this way 8021q and macvlan know
6176 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02006177 */
6178 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00006179 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
Eric W. Biedermand2237d32011-10-21 06:24:20 +00006180 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006181
6182 /*
6183 * Flush the unicast and multicast chains
6184 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006185 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00006186 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006187
6188 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006189 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006190
Eric W. Biedermance286d32007-09-12 13:53:49 +02006191 /* If there is an ifindex conflict assign a new one */
6192 if (__dev_get_by_index(net, dev->ifindex)) {
6193 int iflink = (dev->iflink == dev->ifindex);
6194 dev->ifindex = dev_new_index(net);
6195 if (iflink)
6196 dev->iflink = dev->ifindex;
6197 }
6198
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006199 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07006200 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006201 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006202
6203 /* Add the device back in the hashes */
6204 list_netdevice(dev);
6205
6206 /* Notify protocols, that a new device appeared. */
6207 call_netdevice_notifiers(NETDEV_REGISTER, dev);
6208
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006209 /*
6210 * Prevent userspace races by waiting until the network
6211 * device is fully setup before sending notifications.
6212 */
6213 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
6214
Eric W. Biedermance286d32007-09-12 13:53:49 +02006215 synchronize_net();
6216 err = 0;
6217out:
6218 return err;
6219}
Johannes Berg463d0182009-07-14 00:33:35 +02006220EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006221
Linus Torvalds1da177e2005-04-16 15:20:36 -07006222static int dev_cpu_callback(struct notifier_block *nfb,
6223 unsigned long action,
6224 void *ocpu)
6225{
6226 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006227 struct sk_buff *skb;
6228 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6229 struct softnet_data *sd, *oldsd;
6230
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07006231 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006232 return NOTIFY_OK;
6233
6234 local_irq_disable();
6235 cpu = smp_processor_id();
6236 sd = &per_cpu(softnet_data, cpu);
6237 oldsd = &per_cpu(softnet_data, oldcpu);
6238
6239 /* Find end of our completion_queue. */
6240 list_skb = &sd->completion_queue;
6241 while (*list_skb)
6242 list_skb = &(*list_skb)->next;
6243 /* Append completion queue from offline CPU. */
6244 *list_skb = oldsd->completion_queue;
6245 oldsd->completion_queue = NULL;
6246
Linus Torvalds1da177e2005-04-16 15:20:36 -07006247 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00006248 if (oldsd->output_queue) {
6249 *sd->output_queue_tailp = oldsd->output_queue;
6250 sd->output_queue_tailp = oldsd->output_queue_tailp;
6251 oldsd->output_queue = NULL;
6252 oldsd->output_queue_tailp = &oldsd->output_queue;
6253 }
Heiko Carstens264524d2011-06-06 20:50:03 +00006254 /* Append NAPI poll list from offline CPU. */
6255 if (!list_empty(&oldsd->poll_list)) {
6256 list_splice_init(&oldsd->poll_list, &sd->poll_list);
6257 raise_softirq_irqoff(NET_RX_SOFTIRQ);
6258 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006259
6260 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6261 local_irq_enable();
6262
6263 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00006264 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6265 netif_rx(skb);
6266 input_queue_head_incr(oldsd);
6267 }
Tom Herbertfec5e652010-04-16 16:01:27 -07006268 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006269 netif_rx(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00006270 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07006271 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006272
6273 return NOTIFY_OK;
6274}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006275
6276
Herbert Xu7f353bf2007-08-10 15:47:58 -07006277/**
Herbert Xub63365a2008-10-23 01:11:29 -07006278 * netdev_increment_features - increment feature set by one
6279 * @all: current feature set
6280 * @one: new feature set
6281 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07006282 *
6283 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07006284 * @one to the master device with current feature set @all. Will not
6285 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07006286 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006287netdev_features_t netdev_increment_features(netdev_features_t all,
6288 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07006289{
Michał Mirosław1742f182011-04-22 06:31:16 +00006290 if (mask & NETIF_F_GEN_CSUM)
6291 mask |= NETIF_F_ALL_CSUM;
6292 mask |= NETIF_F_VLAN_CHALLENGED;
6293
6294 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6295 all &= one | ~NETIF_F_ALL_FOR_ALL;
6296
Michał Mirosław1742f182011-04-22 06:31:16 +00006297 /* If one device supports hw checksumming, set for all. */
6298 if (all & NETIF_F_GEN_CSUM)
6299 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006300
6301 return all;
6302}
Herbert Xub63365a2008-10-23 01:11:29 -07006303EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006304
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006305static struct hlist_head *netdev_create_hash(void)
6306{
6307 int i;
6308 struct hlist_head *hash;
6309
6310 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6311 if (hash != NULL)
6312 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6313 INIT_HLIST_HEAD(&hash[i]);
6314
6315 return hash;
6316}
6317
Eric W. Biederman881d9662007-09-17 11:56:21 -07006318/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07006319static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006320{
Rustad, Mark D734b6542012-07-18 09:06:07 +00006321 if (net != &init_net)
6322 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07006323
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006324 net->dev_name_head = netdev_create_hash();
6325 if (net->dev_name_head == NULL)
6326 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006327
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006328 net->dev_index_head = netdev_create_hash();
6329 if (net->dev_index_head == NULL)
6330 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006331
6332 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006333
6334err_idx:
6335 kfree(net->dev_name_head);
6336err_name:
6337 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006338}
6339
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006340/**
6341 * netdev_drivername - network driver for the device
6342 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006343 *
6344 * Determine network driver for device.
6345 */
David S. Miller3019de12011-06-06 16:41:33 -07006346const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07006347{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006348 const struct device_driver *driver;
6349 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07006350 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07006351
6352 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006353 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07006354 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006355
6356 driver = parent->driver;
6357 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07006358 return driver->name;
6359 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006360}
6361
Jason Baronffa10cb2011-08-11 14:36:48 -04006362int __netdev_printk(const char *level, const struct net_device *dev,
Joe Perches256df2f2010-06-27 01:02:35 +00006363 struct va_format *vaf)
6364{
6365 int r;
6366
6367 if (dev && dev->dev.parent)
6368 r = dev_printk(level, dev->dev.parent, "%s: %pV",
6369 netdev_name(dev), vaf);
6370 else if (dev)
6371 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6372 else
6373 r = printk("%s(NULL net_device): %pV", level, vaf);
6374
6375 return r;
6376}
Jason Baronffa10cb2011-08-11 14:36:48 -04006377EXPORT_SYMBOL(__netdev_printk);
Joe Perches256df2f2010-06-27 01:02:35 +00006378
6379int netdev_printk(const char *level, const struct net_device *dev,
6380 const char *format, ...)
6381{
6382 struct va_format vaf;
6383 va_list args;
6384 int r;
6385
6386 va_start(args, format);
6387
6388 vaf.fmt = format;
6389 vaf.va = &args;
6390
6391 r = __netdev_printk(level, dev, &vaf);
6392 va_end(args);
6393
6394 return r;
6395}
6396EXPORT_SYMBOL(netdev_printk);
6397
6398#define define_netdev_printk_level(func, level) \
6399int func(const struct net_device *dev, const char *fmt, ...) \
6400{ \
6401 int r; \
6402 struct va_format vaf; \
6403 va_list args; \
6404 \
6405 va_start(args, fmt); \
6406 \
6407 vaf.fmt = fmt; \
6408 vaf.va = &args; \
6409 \
6410 r = __netdev_printk(level, dev, &vaf); \
6411 va_end(args); \
6412 \
6413 return r; \
6414} \
6415EXPORT_SYMBOL(func);
6416
6417define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6418define_netdev_printk_level(netdev_alert, KERN_ALERT);
6419define_netdev_printk_level(netdev_crit, KERN_CRIT);
6420define_netdev_printk_level(netdev_err, KERN_ERR);
6421define_netdev_printk_level(netdev_warn, KERN_WARNING);
6422define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6423define_netdev_printk_level(netdev_info, KERN_INFO);
6424
Pavel Emelyanov46650792007-10-08 20:38:39 -07006425static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006426{
6427 kfree(net->dev_name_head);
6428 kfree(net->dev_index_head);
6429}
6430
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006431static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006432 .init = netdev_init,
6433 .exit = netdev_exit,
6434};
6435
Pavel Emelyanov46650792007-10-08 20:38:39 -07006436static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006437{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006438 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006439 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006440 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006441 * initial network namespace
6442 */
6443 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006444 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006445 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006446 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006447
6448 /* Ignore unmoveable devices (i.e. loopback) */
6449 if (dev->features & NETIF_F_NETNS_LOCAL)
6450 continue;
6451
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006452 /* Leave virtual devices for the generic cleanup */
6453 if (dev->rtnl_link_ops)
6454 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006455
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006456 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006457 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6458 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006459 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006460 pr_emerg("%s: failed to move %s to init_net: %d\n",
6461 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006462 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006463 }
6464 }
6465 rtnl_unlock();
6466}
6467
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006468static void __net_exit default_device_exit_batch(struct list_head *net_list)
6469{
6470 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04006471 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006472 * Do this across as many network namespaces as possible to
6473 * improve batching efficiency.
6474 */
6475 struct net_device *dev;
6476 struct net *net;
6477 LIST_HEAD(dev_kill_list);
6478
6479 rtnl_lock();
6480 list_for_each_entry(net, net_list, exit_list) {
6481 for_each_netdev_reverse(net, dev) {
6482 if (dev->rtnl_link_ops)
6483 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6484 else
6485 unregister_netdevice_queue(dev, &dev_kill_list);
6486 }
6487 }
6488 unregister_netdevice_many(&dev_kill_list);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006489 list_del(&dev_kill_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006490 rtnl_unlock();
6491}
6492
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006493static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006494 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006495 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006496};
6497
Linus Torvalds1da177e2005-04-16 15:20:36 -07006498/*
6499 * Initialize the DEV module. At boot time this walks the device list and
6500 * unhooks any devices that fail to initialise (normally hardware not
6501 * present) and leaves us with a valid list of present and active devices.
6502 *
6503 */
6504
6505/*
6506 * This is called single threaded during boot, so no need
6507 * to take the rtnl semaphore.
6508 */
6509static int __init net_dev_init(void)
6510{
6511 int i, rc = -ENOMEM;
6512
6513 BUG_ON(!dev_boot_phase);
6514
Linus Torvalds1da177e2005-04-16 15:20:36 -07006515 if (dev_proc_init())
6516 goto out;
6517
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006518 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006519 goto out;
6520
6521 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006522 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006523 INIT_LIST_HEAD(&ptype_base[i]);
6524
Eric W. Biederman881d9662007-09-17 11:56:21 -07006525 if (register_pernet_subsys(&netdev_net_ops))
6526 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006527
6528 /*
6529 * Initialise the packet receive queues.
6530 */
6531
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006532 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006533 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006534
Changli Gaodee42872010-05-02 05:42:16 +00006535 memset(sd, 0, sizeof(*sd));
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006536 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006537 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006538 sd->completion_queue = NULL;
6539 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00006540 sd->output_queue = NULL;
6541 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00006542#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006543 sd->csd.func = rps_trigger_softirq;
6544 sd->csd.info = sd;
6545 sd->csd.flags = 0;
6546 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07006547#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006548
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006549 sd->backlog.poll = process_backlog;
6550 sd->backlog.weight = weight_p;
6551 sd->backlog.gro_list = NULL;
6552 sd->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006553 }
6554
Linus Torvalds1da177e2005-04-16 15:20:36 -07006555 dev_boot_phase = 0;
6556
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006557 /* The loopback device is special if any other network devices
6558 * is present in a network namespace the loopback device must
6559 * be present. Since we now dynamically allocate and free the
6560 * loopback device ensure this invariant is maintained by
6561 * keeping the loopback device as the first device on the
6562 * list of network devices. Ensuring the loopback devices
6563 * is the first device that appears and the last network device
6564 * that disappears.
6565 */
6566 if (register_pernet_device(&loopback_net_ops))
6567 goto out;
6568
6569 if (register_pernet_device(&default_device_ops))
6570 goto out;
6571
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006572 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6573 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006574
6575 hotcpu_notifier(dev_cpu_callback, 0);
6576 dst_init();
6577 dev_mcast_init();
6578 rc = 0;
6579out:
6580 return rc;
6581}
6582
6583subsys_initcall(net_dev_init);
6584
Krishna Kumare88721f2009-02-18 17:55:02 -08006585static int __init initialize_hashrnd(void)
6586{
Tom Herbert0a9627f2010-03-16 08:03:29 +00006587 get_random_bytes(&hashrnd, sizeof(hashrnd));
Krishna Kumare88721f2009-02-18 17:55:02 -08006588 return 0;
6589}
6590
6591late_initcall_sync(initialize_hashrnd);
6592