blob: 81442957c5c2f06669f7c848e21b2b3fdf42ba80 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
76#include <asm/system.h>
77#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080078#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#include <linux/cpu.h>
80#include <linux/types.h>
81#include <linux/kernel.h>
82#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080083#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070084#include <linux/string.h>
85#include <linux/mm.h>
86#include <linux/socket.h>
87#include <linux/sockios.h>
88#include <linux/errno.h>
89#include <linux/interrupt.h>
90#include <linux/if_ether.h>
91#include <linux/netdevice.h>
92#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070093#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070094#include <linux/notifier.h>
95#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020096#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070097#include <net/sock.h>
98#include <linux/rtnetlink.h>
99#include <linux/proc_fs.h>
100#include <linux/seq_file.h>
101#include <linux/stat.h>
102#include <linux/if_bridge.h>
Patrick McHardyb863ceb2007-07-14 18:55:06 -0700103#include <linux/if_macvlan.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104#include <net/dst.h>
105#include <net/pkt_sched.h>
106#include <net/checksum.h>
107#include <linux/highmem.h>
108#include <linux/init.h>
109#include <linux/kmod.h>
110#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <linux/netpoll.h>
112#include <linux/rcupdate.h>
113#include <linux/delay.h>
Johannes Berg295f4a12007-04-26 20:43:56 -0700114#include <net/wext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500117#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700118#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700119#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700120#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700121#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700122#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700123#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700124#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700125#include <linux/ipv6.h>
126#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700127#include <linux/jhash.h>
128#include <linux/random.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700130#include "net-sysfs.h"
131
Herbert Xud565b0a2008-12-15 23:38:52 -0800132/* Instead of increasing this, you should create a hash table. */
133#define MAX_GRO_SKBS 8
134
Herbert Xu5d38a072009-01-04 16:13:40 -0800135/* This should be increased if a protocol with a bigger head is added. */
136#define GRO_MAX_HEAD (MAX_HEADER + 128)
137
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138/*
139 * The list of packet types we will receive (as opposed to discard)
140 * and the routines to invoke.
141 *
142 * Why 16. Because with 16 the only overlap we get on a hash of the
143 * low nibble of the protocol value is RARP/SNAP/X.25.
144 *
145 * NOTE: That is no longer true with the addition of VLAN tags. Not
146 * sure which should go first, but I bet it won't make much
147 * difference if we are running VLANs. The good news is that
148 * this protocol won't be in the list unless compiled in, so
Stephen Hemminger3041a062006-05-26 13:25:24 -0700149 * the average user (w/out VLANs) will not be adversely affected.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150 * --BLG
151 *
152 * 0800 IP
153 * 8100 802.1Q VLAN
154 * 0001 802.3
155 * 0002 AX.25
156 * 0004 802.2
157 * 8035 RARP
158 * 0005 SNAP
159 * 0805 X.25
160 * 0806 ARP
161 * 8137 IPX
162 * 0009 Localtalk
163 * 86DD IPv6
164 */
165
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800166#define PTYPE_HASH_SIZE (16)
167#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
168
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169static DEFINE_SPINLOCK(ptype_lock);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800170static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -0700171static struct list_head ptype_all __read_mostly; /* Taps */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700174 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 * semaphore.
176 *
177 * Pure readers hold dev_base_lock for reading.
178 *
179 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700180 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 * actual updates. This allows pure readers to access the list even
182 * while a writer is preparing to update it.
183 *
184 * To put it another way, dev_base_lock is held for writing only to
185 * protect against pure readers; the rtnl semaphore provides the
186 * protection against other writers.
187 *
188 * See, for example usages, register_netdevice() and
189 * unregister_netdevice(), which must be called with the rtnl
190 * semaphore held.
191 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192DEFINE_RWLOCK(dev_base_lock);
193
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194EXPORT_SYMBOL(dev_base_lock);
195
196#define NETDEV_HASHBITS 8
Eric W. Biederman881d9662007-09-17 11:56:21 -0700197#define NETDEV_HASHENTRIES (1 << NETDEV_HASHBITS)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198
Eric W. Biederman881d9662007-09-17 11:56:21 -0700199static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200{
201 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
Eric W. Biederman881d9662007-09-17 11:56:21 -0700202 return &net->dev_name_head[hash & ((1 << NETDEV_HASHBITS) - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203}
204
Eric W. Biederman881d9662007-09-17 11:56:21 -0700205static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206{
Eric W. Biederman881d9662007-09-17 11:56:21 -0700207 return &net->dev_index_head[ifindex & ((1 << NETDEV_HASHBITS) - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208}
209
Eric W. Biedermance286d32007-09-12 13:53:49 +0200210/* Device list insertion */
211static int list_netdevice(struct net_device *dev)
212{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900213 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200214
215 ASSERT_RTNL();
216
217 write_lock_bh(&dev_base_lock);
218 list_add_tail(&dev->dev_list, &net->dev_base_head);
219 hlist_add_head(&dev->name_hlist, dev_name_hash(net, dev->name));
220 hlist_add_head(&dev->index_hlist, dev_index_hash(net, dev->ifindex));
221 write_unlock_bh(&dev_base_lock);
222 return 0;
223}
224
225/* Device list removal */
226static void unlist_netdevice(struct net_device *dev)
227{
228 ASSERT_RTNL();
229
230 /* Unlink dev from the device chain */
231 write_lock_bh(&dev_base_lock);
232 list_del(&dev->dev_list);
233 hlist_del(&dev->name_hlist);
234 hlist_del(&dev->index_hlist);
235 write_unlock_bh(&dev_base_lock);
236}
237
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238/*
239 * Our notifier list
240 */
241
Alan Sternf07d5b92006-05-09 15:23:03 -0700242static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243
244/*
245 * Device drivers call our routines to queue packets here. We empty the
246 * queue in the local softnet handler.
247 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700248
249DEFINE_PER_CPU(struct softnet_data, softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250
David S. Millercf508b12008-07-22 14:16:42 -0700251#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700252/*
David S. Millerc773e842008-07-08 23:13:53 -0700253 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700254 * according to dev->type
255 */
256static const unsigned short netdev_lock_type[] =
257 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
258 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
259 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
260 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
261 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
262 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
263 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
264 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
265 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
266 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
267 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
268 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
269 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800270 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
Rémi Denis-Courmont57c81ff2008-12-17 15:47:48 -0800271 ARPHRD_PHONET_PIPE, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700272
273static const char *netdev_lock_name[] =
274 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
275 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
276 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
277 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
278 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
279 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
280 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
281 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
282 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
283 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
284 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
285 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
286 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800287 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
Rémi Denis-Courmont57c81ff2008-12-17 15:47:48 -0800288 "_xmit_PHONET_PIPE", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700289
290static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700291static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700292
293static inline unsigned short netdev_lock_pos(unsigned short dev_type)
294{
295 int i;
296
297 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
298 if (netdev_lock_type[i] == dev_type)
299 return i;
300 /* the last key is used by default */
301 return ARRAY_SIZE(netdev_lock_type) - 1;
302}
303
David S. Millercf508b12008-07-22 14:16:42 -0700304static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
305 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700306{
307 int i;
308
309 i = netdev_lock_pos(dev_type);
310 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
311 netdev_lock_name[i]);
312}
David S. Millercf508b12008-07-22 14:16:42 -0700313
314static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
315{
316 int i;
317
318 i = netdev_lock_pos(dev->type);
319 lockdep_set_class_and_name(&dev->addr_list_lock,
320 &netdev_addr_lock_key[i],
321 netdev_lock_name[i]);
322}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700323#else
David S. Millercf508b12008-07-22 14:16:42 -0700324static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
325 unsigned short dev_type)
326{
327}
328static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700329{
330}
331#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332
333/*******************************************************************************
334
335 Protocol management and registration routines
336
337*******************************************************************************/
338
339/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 * Add a protocol ID to the list. Now that the input handler is
341 * smarter we can dispense with all the messy stuff that used to be
342 * here.
343 *
344 * BEWARE!!! Protocol handlers, mangling input packets,
345 * MUST BE last in hash buckets and checking protocol handlers
346 * MUST start from promiscuous ptype_all chain in net_bh.
347 * It is true now, do not change it.
348 * Explanation follows: if protocol handler, mangling packet, will
349 * be the first on list, it is not able to sense, that packet
350 * is cloned and should be copied-on-write, so that it will
351 * change it and subsequent readers will get broken packet.
352 * --ANK (980803)
353 */
354
355/**
356 * dev_add_pack - add packet handler
357 * @pt: packet type declaration
358 *
359 * Add a protocol handler to the networking stack. The passed &packet_type
360 * is linked into kernel lists and may not be freed until it has been
361 * removed from the kernel lists.
362 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900363 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 * guarantee all CPU's that are in middle of receiving packets
365 * will see the new packet type (until the next received packet).
366 */
367
368void dev_add_pack(struct packet_type *pt)
369{
370 int hash;
371
372 spin_lock_bh(&ptype_lock);
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -0700373 if (pt->type == htons(ETH_P_ALL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 list_add_rcu(&pt->list, &ptype_all);
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -0700375 else {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800376 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 list_add_rcu(&pt->list, &ptype_base[hash]);
378 }
379 spin_unlock_bh(&ptype_lock);
380}
381
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382/**
383 * __dev_remove_pack - remove packet handler
384 * @pt: packet type declaration
385 *
386 * Remove a protocol handler that was previously added to the kernel
387 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
388 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900389 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 *
391 * The packet type might still be in use by receivers
392 * and must not be freed until after all the CPU's have gone
393 * through a quiescent state.
394 */
395void __dev_remove_pack(struct packet_type *pt)
396{
397 struct list_head *head;
398 struct packet_type *pt1;
399
400 spin_lock_bh(&ptype_lock);
401
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -0700402 if (pt->type == htons(ETH_P_ALL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403 head = &ptype_all;
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -0700404 else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800405 head = &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406
407 list_for_each_entry(pt1, head, list) {
408 if (pt == pt1) {
409 list_del_rcu(&pt->list);
410 goto out;
411 }
412 }
413
414 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
415out:
416 spin_unlock_bh(&ptype_lock);
417}
418/**
419 * dev_remove_pack - remove packet handler
420 * @pt: packet type declaration
421 *
422 * Remove a protocol handler that was previously added to the kernel
423 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
424 * from the kernel lists and can be freed or reused once this function
425 * returns.
426 *
427 * This call sleeps to guarantee that no CPU is looking at the packet
428 * type after return.
429 */
430void dev_remove_pack(struct packet_type *pt)
431{
432 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900433
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 synchronize_net();
435}
436
437/******************************************************************************
438
439 Device Boot-time Settings Routines
440
441*******************************************************************************/
442
443/* Boot time configuration table */
444static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
445
446/**
447 * netdev_boot_setup_add - add new setup entry
448 * @name: name of the device
449 * @map: configured settings for the device
450 *
451 * Adds new setup entry to the dev_boot_setup list. The function
452 * returns 0 on error and 1 on success. This is a generic routine to
453 * all netdevices.
454 */
455static int netdev_boot_setup_add(char *name, struct ifmap *map)
456{
457 struct netdev_boot_setup *s;
458 int i;
459
460 s = dev_boot_setup;
461 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
462 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
463 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700464 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 memcpy(&s[i].map, map, sizeof(s[i].map));
466 break;
467 }
468 }
469
470 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
471}
472
473/**
474 * netdev_boot_setup_check - check boot time settings
475 * @dev: the netdevice
476 *
477 * Check boot time settings for the device.
478 * The found settings are set for the device to be used
479 * later in the device probing.
480 * Returns 0 if no settings found, 1 if they are.
481 */
482int netdev_boot_setup_check(struct net_device *dev)
483{
484 struct netdev_boot_setup *s = dev_boot_setup;
485 int i;
486
487 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
488 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700489 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 dev->irq = s[i].map.irq;
491 dev->base_addr = s[i].map.base_addr;
492 dev->mem_start = s[i].map.mem_start;
493 dev->mem_end = s[i].map.mem_end;
494 return 1;
495 }
496 }
497 return 0;
498}
499
500
501/**
502 * netdev_boot_base - get address from boot time settings
503 * @prefix: prefix for network device
504 * @unit: id for network device
505 *
506 * Check boot time settings for the base address of device.
507 * The found settings are set for the device to be used
508 * later in the device probing.
509 * Returns 0 if no settings found.
510 */
511unsigned long netdev_boot_base(const char *prefix, int unit)
512{
513 const struct netdev_boot_setup *s = dev_boot_setup;
514 char name[IFNAMSIZ];
515 int i;
516
517 sprintf(name, "%s%d", prefix, unit);
518
519 /*
520 * If device already registered then return base of 1
521 * to indicate not to probe for this interface
522 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700523 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 return 1;
525
526 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
527 if (!strcmp(name, s[i].name))
528 return s[i].map.base_addr;
529 return 0;
530}
531
532/*
533 * Saves at boot time configured settings for any netdevice.
534 */
535int __init netdev_boot_setup(char *str)
536{
537 int ints[5];
538 struct ifmap map;
539
540 str = get_options(str, ARRAY_SIZE(ints), ints);
541 if (!str || !*str)
542 return 0;
543
544 /* Save settings */
545 memset(&map, 0, sizeof(map));
546 if (ints[0] > 0)
547 map.irq = ints[1];
548 if (ints[0] > 1)
549 map.base_addr = ints[2];
550 if (ints[0] > 2)
551 map.mem_start = ints[3];
552 if (ints[0] > 3)
553 map.mem_end = ints[4];
554
555 /* Add new entry to the list */
556 return netdev_boot_setup_add(str, &map);
557}
558
559__setup("netdev=", netdev_boot_setup);
560
561/*******************************************************************************
562
563 Device Interface Subroutines
564
565*******************************************************************************/
566
567/**
568 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700569 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 * @name: name to find
571 *
572 * Find an interface by name. Must be called under RTNL semaphore
573 * or @dev_base_lock. If the name is found a pointer to the device
574 * is returned. If the name is not found then %NULL is returned. The
575 * reference counters are not incremented so the caller must be
576 * careful with locks.
577 */
578
Eric W. Biederman881d9662007-09-17 11:56:21 -0700579struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580{
581 struct hlist_node *p;
582
Eric W. Biederman881d9662007-09-17 11:56:21 -0700583 hlist_for_each(p, dev_name_hash(net, name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 struct net_device *dev
585 = hlist_entry(p, struct net_device, name_hlist);
586 if (!strncmp(dev->name, name, IFNAMSIZ))
587 return dev;
588 }
589 return NULL;
590}
591
592/**
593 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700594 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 * @name: name to find
596 *
597 * Find an interface by name. This can be called from any
598 * context and does its own locking. The returned handle has
599 * the usage count incremented and the caller must use dev_put() to
600 * release it when it is no longer needed. %NULL is returned if no
601 * matching device is found.
602 */
603
Eric W. Biederman881d9662007-09-17 11:56:21 -0700604struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605{
606 struct net_device *dev;
607
608 read_lock(&dev_base_lock);
Eric W. Biederman881d9662007-09-17 11:56:21 -0700609 dev = __dev_get_by_name(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 if (dev)
611 dev_hold(dev);
612 read_unlock(&dev_base_lock);
613 return dev;
614}
615
616/**
617 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700618 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 * @ifindex: index of device
620 *
621 * Search for an interface by index. Returns %NULL if the device
622 * is not found or a pointer to the device. The device has not
623 * had its reference counter increased so the caller must be careful
624 * about locking. The caller must hold either the RTNL semaphore
625 * or @dev_base_lock.
626 */
627
Eric W. Biederman881d9662007-09-17 11:56:21 -0700628struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629{
630 struct hlist_node *p;
631
Eric W. Biederman881d9662007-09-17 11:56:21 -0700632 hlist_for_each(p, dev_index_hash(net, ifindex)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 struct net_device *dev
634 = hlist_entry(p, struct net_device, index_hlist);
635 if (dev->ifindex == ifindex)
636 return dev;
637 }
638 return NULL;
639}
640
641
642/**
643 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700644 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645 * @ifindex: index of device
646 *
647 * Search for an interface by index. Returns NULL if the device
648 * is not found or a pointer to the device. The device returned has
649 * had a reference added and the pointer is safe until the user calls
650 * dev_put to indicate they have finished with it.
651 */
652
Eric W. Biederman881d9662007-09-17 11:56:21 -0700653struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654{
655 struct net_device *dev;
656
657 read_lock(&dev_base_lock);
Eric W. Biederman881d9662007-09-17 11:56:21 -0700658 dev = __dev_get_by_index(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 if (dev)
660 dev_hold(dev);
661 read_unlock(&dev_base_lock);
662 return dev;
663}
664
665/**
666 * dev_getbyhwaddr - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700667 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 * @type: media type of device
669 * @ha: hardware address
670 *
671 * Search for an interface by MAC address. Returns NULL if the device
672 * is not found or a pointer to the device. The caller must hold the
673 * rtnl semaphore. The returned device has not had its ref count increased
674 * and the caller must therefore be careful about locking
675 *
676 * BUGS:
677 * If the API was consistent this would be __dev_get_by_hwaddr
678 */
679
Eric W. Biederman881d9662007-09-17 11:56:21 -0700680struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681{
682 struct net_device *dev;
683
684 ASSERT_RTNL();
685
Denis V. Lunev81103a52007-12-12 10:47:38 -0800686 for_each_netdev(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 if (dev->type == type &&
688 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700689 return dev;
690
691 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692}
693
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300694EXPORT_SYMBOL(dev_getbyhwaddr);
695
Eric W. Biederman881d9662007-09-17 11:56:21 -0700696struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700697{
698 struct net_device *dev;
699
700 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700701 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700702 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700703 return dev;
704
705 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700706}
707
708EXPORT_SYMBOL(__dev_getfirstbyhwtype);
709
Eric W. Biederman881d9662007-09-17 11:56:21 -0700710struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711{
712 struct net_device *dev;
713
714 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700715 dev = __dev_getfirstbyhwtype(net, type);
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700716 if (dev)
717 dev_hold(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 rtnl_unlock();
719 return dev;
720}
721
722EXPORT_SYMBOL(dev_getfirstbyhwtype);
723
724/**
725 * dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700726 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727 * @if_flags: IFF_* values
728 * @mask: bitmask of bits in if_flags to check
729 *
730 * Search for any interface with the given flags. Returns NULL if a device
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900731 * is not found or a pointer to the device. The device returned has
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732 * had a reference added and the pointer is safe until the user calls
733 * dev_put to indicate they have finished with it.
734 */
735
Eric W. Biederman881d9662007-09-17 11:56:21 -0700736struct net_device * dev_get_by_flags(struct net *net, unsigned short if_flags, unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700738 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739
Pavel Emelianov7562f872007-05-03 15:13:45 -0700740 ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 read_lock(&dev_base_lock);
Eric W. Biederman881d9662007-09-17 11:56:21 -0700742 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 if (((dev->flags ^ if_flags) & mask) == 0) {
744 dev_hold(dev);
Pavel Emelianov7562f872007-05-03 15:13:45 -0700745 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 break;
747 }
748 }
749 read_unlock(&dev_base_lock);
Pavel Emelianov7562f872007-05-03 15:13:45 -0700750 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751}
752
753/**
754 * dev_valid_name - check if name is okay for network device
755 * @name: name string
756 *
757 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700758 * to allow sysfs to work. We also disallow any kind of
759 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760 */
Mitch Williamsc2373ee2005-11-09 10:34:45 -0800761int dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700763 if (*name == '\0')
764 return 0;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700765 if (strlen(name) >= IFNAMSIZ)
766 return 0;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700767 if (!strcmp(name, ".") || !strcmp(name, ".."))
768 return 0;
769
770 while (*name) {
771 if (*name == '/' || isspace(*name))
772 return 0;
773 name++;
774 }
775 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776}
777
778/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200779 * __dev_alloc_name - allocate a name for a device
780 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200782 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783 *
784 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700785 * id. It scans list of devices to build up a free map, then chooses
786 * the first empty slot. The caller must hold the dev_base or rtnl lock
787 * while allocating the name and adding the device in order to avoid
788 * duplicates.
789 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
790 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791 */
792
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200793static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794{
795 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796 const char *p;
797 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700798 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 struct net_device *d;
800
801 p = strnchr(name, IFNAMSIZ-1, '%');
802 if (p) {
803 /*
804 * Verify the string as this thing may have come from
805 * the user. There must be either one "%d" and no other "%"
806 * characters.
807 */
808 if (p[1] != 'd' || strchr(p + 2, '%'))
809 return -EINVAL;
810
811 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700812 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813 if (!inuse)
814 return -ENOMEM;
815
Eric W. Biederman881d9662007-09-17 11:56:21 -0700816 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 if (!sscanf(d->name, name, &i))
818 continue;
819 if (i < 0 || i >= max_netdevices)
820 continue;
821
822 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200823 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824 if (!strncmp(buf, d->name, IFNAMSIZ))
825 set_bit(i, inuse);
826 }
827
828 i = find_first_zero_bit(inuse, max_netdevices);
829 free_page((unsigned long) inuse);
830 }
831
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200832 snprintf(buf, IFNAMSIZ, name, i);
833 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835
836 /* It is possible to run out of possible slots
837 * when the name is long and there isn't enough space left
838 * for the digits, or if all bits are used.
839 */
840 return -ENFILE;
841}
842
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200843/**
844 * dev_alloc_name - allocate a name for a device
845 * @dev: device
846 * @name: name format string
847 *
848 * Passed a format string - eg "lt%d" it will try and find a suitable
849 * id. It scans list of devices to build up a free map, then chooses
850 * the first empty slot. The caller must hold the dev_base or rtnl lock
851 * while allocating the name and adding the device in order to avoid
852 * duplicates.
853 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
854 * Returns the number of the unit assigned or a negative errno code.
855 */
856
857int dev_alloc_name(struct net_device *dev, const char *name)
858{
859 char buf[IFNAMSIZ];
860 struct net *net;
861 int ret;
862
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900863 BUG_ON(!dev_net(dev));
864 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200865 ret = __dev_alloc_name(net, name, buf);
866 if (ret >= 0)
867 strlcpy(dev->name, buf, IFNAMSIZ);
868 return ret;
869}
870
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871
872/**
873 * dev_change_name - change name of a device
874 * @dev: device
875 * @newname: name (or format string) must be at least IFNAMSIZ
876 *
877 * Change name of a device, can pass format strings "eth%d".
878 * for wildcarding.
879 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -0700880int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881{
Herbert Xufcc5a032007-07-30 17:03:38 -0700882 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -0700884 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -0700885 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886
887 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900888 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900890 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891 if (dev->flags & IFF_UP)
892 return -EBUSY;
893
894 if (!dev_valid_name(newname))
895 return -EINVAL;
896
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -0700897 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
898 return 0;
899
Herbert Xufcc5a032007-07-30 17:03:38 -0700900 memcpy(oldname, dev->name, IFNAMSIZ);
901
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 if (strchr(newname, '%')) {
903 err = dev_alloc_name(dev, newname);
904 if (err < 0)
905 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 }
Eric W. Biederman881d9662007-09-17 11:56:21 -0700907 else if (__dev_get_by_name(net, newname))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908 return -EEXIST;
909 else
910 strlcpy(dev->name, newname, IFNAMSIZ);
911
Herbert Xufcc5a032007-07-30 17:03:38 -0700912rollback:
Eric W. Biederman38918452008-10-27 17:51:47 -0700913 /* For now only devices in the initial network namespace
914 * are in sysfs.
915 */
916 if (net == &init_net) {
917 ret = device_rename(&dev->dev, dev->name);
918 if (ret) {
919 memcpy(dev->name, oldname, IFNAMSIZ);
920 return ret;
921 }
Stephen Hemmingerdcc99772008-05-14 22:33:38 -0700922 }
Herbert Xu7f988ea2007-07-30 16:35:46 -0700923
924 write_lock_bh(&dev_base_lock);
Eric W. Biederman92749822007-04-03 00:07:30 -0600925 hlist_del(&dev->name_hlist);
Eric W. Biederman881d9662007-09-17 11:56:21 -0700926 hlist_add_head(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -0700927 write_unlock_bh(&dev_base_lock);
928
Pavel Emelyanov056925a2007-09-16 15:42:43 -0700929 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -0700930 ret = notifier_to_errno(ret);
931
932 if (ret) {
933 if (err) {
934 printk(KERN_ERR
935 "%s: name change rollback failed: %d.\n",
936 dev->name, ret);
937 } else {
938 err = ret;
939 memcpy(dev->name, oldname, IFNAMSIZ);
940 goto rollback;
941 }
942 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943
944 return err;
945}
946
947/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -0700948 * dev_set_alias - change ifalias of a device
949 * @dev: device
950 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -0700951 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -0700952 *
953 * Set ifalias for a device,
954 */
955int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
956{
957 ASSERT_RTNL();
958
959 if (len >= IFALIASZ)
960 return -EINVAL;
961
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -0700962 if (!len) {
963 if (dev->ifalias) {
964 kfree(dev->ifalias);
965 dev->ifalias = NULL;
966 }
967 return 0;
968 }
969
Stephen Hemminger0b815a12008-09-22 21:28:11 -0700970 dev->ifalias = krealloc(dev->ifalias, len+1, GFP_KERNEL);
971 if (!dev->ifalias)
972 return -ENOMEM;
973
974 strlcpy(dev->ifalias, alias, len+1);
975 return len;
976}
977
978
979/**
Stephen Hemminger3041a062006-05-26 13:25:24 -0700980 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -0700981 * @dev: device to cause notification
982 *
983 * Called to indicate a device has changed features.
984 */
985void netdev_features_change(struct net_device *dev)
986{
Pavel Emelyanov056925a2007-09-16 15:42:43 -0700987 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -0700988}
989EXPORT_SYMBOL(netdev_features_change);
990
991/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 * netdev_state_change - device changes state
993 * @dev: device to cause notification
994 *
995 * Called to indicate a device has changed state. This function calls
996 * the notifier chains for netdev_chain and sends a NEWLINK message
997 * to the routing socket.
998 */
999void netdev_state_change(struct net_device *dev)
1000{
1001 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001002 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1004 }
1005}
1006
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001007void netdev_bonding_change(struct net_device *dev)
1008{
1009 call_netdevice_notifiers(NETDEV_BONDING_FAILOVER, dev);
1010}
1011EXPORT_SYMBOL(netdev_bonding_change);
1012
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013/**
1014 * dev_load - load a network module
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001015 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016 * @name: name of interface
1017 *
1018 * If a network interface is not present and the process has suitable
1019 * privileges this function loads the module. If module loading is not
1020 * available in this kernel then it becomes a nop.
1021 */
1022
Eric W. Biederman881d9662007-09-17 11:56:21 -07001023void dev_load(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024{
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001025 struct net_device *dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026
1027 read_lock(&dev_base_lock);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001028 dev = __dev_get_by_name(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029 read_unlock(&dev_base_lock);
1030
1031 if (!dev && capable(CAP_SYS_MODULE))
1032 request_module("%s", name);
1033}
1034
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035/**
1036 * dev_open - prepare an interface for use.
1037 * @dev: device to open
1038 *
1039 * Takes a device from down to up state. The device's private open
1040 * function is invoked and then the multicast lists are loaded. Finally
1041 * the device is moved into the up state and a %NETDEV_UP message is
1042 * sent to the netdev notifier chain.
1043 *
1044 * Calling this function on an active interface is a nop. On a failure
1045 * a negative errno code is returned.
1046 */
1047int dev_open(struct net_device *dev)
1048{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001049 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050 int ret = 0;
1051
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001052 ASSERT_RTNL();
1053
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054 /*
1055 * Is it already up?
1056 */
1057
1058 if (dev->flags & IFF_UP)
1059 return 0;
1060
1061 /*
1062 * Is it even present?
1063 */
1064 if (!netif_device_present(dev))
1065 return -ENODEV;
1066
1067 /*
1068 * Call device private open method
1069 */
1070 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001071
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001072 if (ops->ndo_validate_addr)
1073 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001074
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001075 if (!ret && ops->ndo_open)
1076 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001078 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079 * If it went open OK then:
1080 */
1081
Jeff Garzikbada3392007-10-23 20:19:37 -07001082 if (ret)
1083 clear_bit(__LINK_STATE_START, &dev->state);
1084 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085 /*
1086 * Set the flags.
1087 */
1088 dev->flags |= IFF_UP;
1089
1090 /*
Dan Williams649274d2009-01-11 00:20:39 -08001091 * Enable NET_DMA
1092 */
David S. Millerb4bd07c2009-02-06 22:06:43 -08001093 net_dmaengine_get();
Dan Williams649274d2009-01-11 00:20:39 -08001094
1095 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096 * Initialize multicasting status
1097 */
Patrick McHardy4417da62007-06-27 01:28:10 -07001098 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099
1100 /*
1101 * Wakeup transmit queue engine
1102 */
1103 dev_activate(dev);
1104
1105 /*
1106 * ... and announce new interface.
1107 */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001108 call_netdevice_notifiers(NETDEV_UP, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001110
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111 return ret;
1112}
1113
1114/**
1115 * dev_close - shutdown an interface.
1116 * @dev: device to shutdown
1117 *
1118 * This function moves an active device into down state. A
1119 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1120 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1121 * chain.
1122 */
1123int dev_close(struct net_device *dev)
1124{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001125 const struct net_device_ops *ops = dev->netdev_ops;
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001126 ASSERT_RTNL();
1127
David S. Miller9d5010d2007-09-12 14:33:25 +02001128 might_sleep();
1129
Linus Torvalds1da177e2005-04-16 15:20:36 -07001130 if (!(dev->flags & IFF_UP))
1131 return 0;
1132
1133 /*
1134 * Tell people we are going down, so that they can
1135 * prepare to death, when device is still operating.
1136 */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001137 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139 clear_bit(__LINK_STATE_START, &dev->state);
1140
1141 /* Synchronize to scheduled poll. We cannot touch poll list,
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001142 * it can be even on different cpu. So just clear netif_running().
1143 *
1144 * dev->stop() will invoke napi_disable() on all of it's
1145 * napi_struct instances on this device.
1146 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147 smp_mb__after_clear_bit(); /* Commit netif_running(). */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001149 dev_deactivate(dev);
1150
Linus Torvalds1da177e2005-04-16 15:20:36 -07001151 /*
1152 * Call the device specific close. This cannot fail.
1153 * Only if device is UP
1154 *
1155 * We allow it to be called even after a DETACH hot-plug
1156 * event.
1157 */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001158 if (ops->ndo_stop)
1159 ops->ndo_stop(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160
1161 /*
1162 * Device is now down.
1163 */
1164
1165 dev->flags &= ~IFF_UP;
1166
1167 /*
1168 * Tell people we are down
1169 */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001170 call_netdevice_notifiers(NETDEV_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171
Dan Williams649274d2009-01-11 00:20:39 -08001172 /*
1173 * Shutdown NET_DMA
1174 */
David S. Millerb4bd07c2009-02-06 22:06:43 -08001175 net_dmaengine_put();
Dan Williams649274d2009-01-11 00:20:39 -08001176
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177 return 0;
1178}
1179
1180
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001181/**
1182 * dev_disable_lro - disable Large Receive Offload on a device
1183 * @dev: device
1184 *
1185 * Disable Large Receive Offload (LRO) on a net device. Must be
1186 * called under RTNL. This is needed if received packets may be
1187 * forwarded to another interface.
1188 */
1189void dev_disable_lro(struct net_device *dev)
1190{
1191 if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&
1192 dev->ethtool_ops->set_flags) {
1193 u32 flags = dev->ethtool_ops->get_flags(dev);
1194 if (flags & ETH_FLAG_LRO) {
1195 flags &= ~ETH_FLAG_LRO;
1196 dev->ethtool_ops->set_flags(dev, flags);
1197 }
1198 }
1199 WARN_ON(dev->features & NETIF_F_LRO);
1200}
1201EXPORT_SYMBOL(dev_disable_lro);
1202
1203
Eric W. Biederman881d9662007-09-17 11:56:21 -07001204static int dev_boot_phase = 1;
1205
Linus Torvalds1da177e2005-04-16 15:20:36 -07001206/*
1207 * Device change register/unregister. These are not inline or static
1208 * as we export them to the world.
1209 */
1210
1211/**
1212 * register_netdevice_notifier - register a network notifier block
1213 * @nb: notifier
1214 *
1215 * Register a notifier to be called when network device events occur.
1216 * The notifier passed is linked into the kernel structures and must
1217 * not be reused until it has been unregistered. A negative errno code
1218 * is returned on a failure.
1219 *
1220 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001221 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222 * view of the network device list.
1223 */
1224
1225int register_netdevice_notifier(struct notifier_block *nb)
1226{
1227 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001228 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001229 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230 int err;
1231
1232 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001233 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001234 if (err)
1235 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001236 if (dev_boot_phase)
1237 goto unlock;
1238 for_each_net(net) {
1239 for_each_netdev(net, dev) {
1240 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1241 err = notifier_to_errno(err);
1242 if (err)
1243 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244
Eric W. Biederman881d9662007-09-17 11:56:21 -07001245 if (!(dev->flags & IFF_UP))
1246 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001247
Eric W. Biederman881d9662007-09-17 11:56:21 -07001248 nb->notifier_call(nb, NETDEV_UP, dev);
1249 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001251
1252unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253 rtnl_unlock();
1254 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001255
1256rollback:
1257 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001258 for_each_net(net) {
1259 for_each_netdev(net, dev) {
1260 if (dev == last)
1261 break;
Herbert Xufcc5a032007-07-30 17:03:38 -07001262
Eric W. Biederman881d9662007-09-17 11:56:21 -07001263 if (dev->flags & IFF_UP) {
1264 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1265 nb->notifier_call(nb, NETDEV_DOWN, dev);
1266 }
1267 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001268 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001269 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001270
1271 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001272 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001273}
1274
1275/**
1276 * unregister_netdevice_notifier - unregister a network notifier block
1277 * @nb: notifier
1278 *
1279 * Unregister a notifier previously registered by
1280 * register_netdevice_notifier(). The notifier is unlinked into the
1281 * kernel structures and may then be reused. A negative errno code
1282 * is returned on a failure.
1283 */
1284
1285int unregister_netdevice_notifier(struct notifier_block *nb)
1286{
Herbert Xu9f514952006-03-25 01:24:25 -08001287 int err;
1288
1289 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001290 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xu9f514952006-03-25 01:24:25 -08001291 rtnl_unlock();
1292 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293}
1294
1295/**
1296 * call_netdevice_notifiers - call all network notifier blocks
1297 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001298 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 *
1300 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001301 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001302 */
1303
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001304int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305{
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001306 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307}
1308
1309/* When > 0 there are consumers of rx skb time stamps */
1310static atomic_t netstamp_needed = ATOMIC_INIT(0);
1311
1312void net_enable_timestamp(void)
1313{
1314 atomic_inc(&netstamp_needed);
1315}
1316
1317void net_disable_timestamp(void)
1318{
1319 atomic_dec(&netstamp_needed);
1320}
1321
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001322static inline void net_timestamp(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323{
1324 if (atomic_read(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001325 __net_timestamp(skb);
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07001326 else
1327 skb->tstamp.tv64 = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328}
1329
1330/*
1331 * Support routine. Sends outgoing frames to any network
1332 * taps currently in use.
1333 */
1334
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001335static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336{
1337 struct packet_type *ptype;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001338
Jarek Poplawski8caf1532009-04-17 10:08:49 +00001339#ifdef CONFIG_NET_CLS_ACT
1340 if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS)))
1341 net_timestamp(skb);
1342#else
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001343 net_timestamp(skb);
Jarek Poplawski8caf1532009-04-17 10:08:49 +00001344#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345
1346 rcu_read_lock();
1347 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1348 /* Never send packets back to the socket
1349 * they originated from - MvS (miquels@drinkel.ow.org)
1350 */
1351 if ((ptype->dev == dev || !ptype->dev) &&
1352 (ptype->af_packet_priv == NULL ||
1353 (struct sock *)ptype->af_packet_priv != skb->sk)) {
1354 struct sk_buff *skb2= skb_clone(skb, GFP_ATOMIC);
1355 if (!skb2)
1356 break;
1357
1358 /* skb->nh should be correctly
1359 set by sender, so that the second statement is
1360 just protection against buggy protocols.
1361 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001362 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001364 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001365 skb2->network_header > skb2->tail) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366 if (net_ratelimit())
1367 printk(KERN_CRIT "protocol %04x is "
1368 "buggy, dev %s\n",
1369 skb2->protocol, dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001370 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371 }
1372
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001373 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374 skb2->pkt_type = PACKET_OUTGOING;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07001375 ptype->func(skb2, skb->dev, ptype, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376 }
1377 }
1378 rcu_read_unlock();
1379}
1380
Denis Vlasenko56079432006-03-29 15:57:29 -08001381
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001382static inline void __netif_reschedule(struct Qdisc *q)
1383{
1384 struct softnet_data *sd;
1385 unsigned long flags;
1386
1387 local_irq_save(flags);
1388 sd = &__get_cpu_var(softnet_data);
1389 q->next_sched = sd->output_queue;
1390 sd->output_queue = q;
1391 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1392 local_irq_restore(flags);
1393}
1394
David S. Miller37437bb2008-07-16 02:15:04 -07001395void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08001396{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001397 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1398 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08001399}
1400EXPORT_SYMBOL(__netif_schedule);
1401
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001402void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08001403{
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001404 if (atomic_dec_and_test(&skb->users)) {
1405 struct softnet_data *sd;
1406 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08001407
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001408 local_irq_save(flags);
1409 sd = &__get_cpu_var(softnet_data);
1410 skb->next = sd->completion_queue;
1411 sd->completion_queue = skb;
1412 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1413 local_irq_restore(flags);
1414 }
Denis Vlasenko56079432006-03-29 15:57:29 -08001415}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001416EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08001417
1418void dev_kfree_skb_any(struct sk_buff *skb)
1419{
1420 if (in_irq() || irqs_disabled())
1421 dev_kfree_skb_irq(skb);
1422 else
1423 dev_kfree_skb(skb);
1424}
1425EXPORT_SYMBOL(dev_kfree_skb_any);
1426
1427
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001428/**
1429 * netif_device_detach - mark device as removed
1430 * @dev: network device
1431 *
1432 * Mark device as removed from system and therefore no longer available.
1433 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001434void netif_device_detach(struct net_device *dev)
1435{
1436 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1437 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001438 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001439 }
1440}
1441EXPORT_SYMBOL(netif_device_detach);
1442
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001443/**
1444 * netif_device_attach - mark device as attached
1445 * @dev: network device
1446 *
1447 * Mark device as attached from system and restart if needed.
1448 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001449void netif_device_attach(struct net_device *dev)
1450{
1451 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1452 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001453 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001454 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001455 }
1456}
1457EXPORT_SYMBOL(netif_device_attach);
1458
Ben Hutchings6de329e2008-06-16 17:02:28 -07001459static bool can_checksum_protocol(unsigned long features, __be16 protocol)
1460{
1461 return ((features & NETIF_F_GEN_CSUM) ||
1462 ((features & NETIF_F_IP_CSUM) &&
1463 protocol == htons(ETH_P_IP)) ||
1464 ((features & NETIF_F_IPV6_CSUM) &&
Yi Zou1c8dbcf2009-02-27 14:06:54 -08001465 protocol == htons(ETH_P_IPV6)) ||
1466 ((features & NETIF_F_FCOE_CRC) &&
1467 protocol == htons(ETH_P_FCOE)));
Ben Hutchings6de329e2008-06-16 17:02:28 -07001468}
1469
1470static bool dev_can_checksum(struct net_device *dev, struct sk_buff *skb)
1471{
1472 if (can_checksum_protocol(dev->features, skb->protocol))
1473 return true;
1474
1475 if (skb->protocol == htons(ETH_P_8021Q)) {
1476 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
1477 if (can_checksum_protocol(dev->features & dev->vlan_features,
1478 veh->h_vlan_encapsulated_proto))
1479 return true;
1480 }
1481
1482 return false;
1483}
Denis Vlasenko56079432006-03-29 15:57:29 -08001484
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485/*
1486 * Invalidate hardware checksum when packet is to be mangled, and
1487 * complete checksum manually on outgoing path.
1488 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07001489int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490{
Al Virod3bc23e2006-11-14 21:24:49 -08001491 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07001492 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001493
Patrick McHardy84fa7932006-08-29 16:44:56 -07001494 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07001495 goto out_set_summed;
1496
1497 if (unlikely(skb_shinfo(skb)->gso_size)) {
Herbert Xua430a432006-07-08 13:34:56 -07001498 /* Let GSO fix up the checksum. */
1499 goto out_set_summed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500 }
1501
Herbert Xua0308472007-10-15 01:47:15 -07001502 offset = skb->csum_start - skb_headroom(skb);
1503 BUG_ON(offset >= skb_headlen(skb));
1504 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1505
1506 offset += skb->csum_offset;
1507 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1508
1509 if (skb_cloned(skb) &&
1510 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1512 if (ret)
1513 goto out;
1514 }
1515
Herbert Xua0308472007-10-15 01:47:15 -07001516 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07001517out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001519out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520 return ret;
1521}
1522
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001523/**
1524 * skb_gso_segment - Perform segmentation on skb.
1525 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07001526 * @features: features for the output path (see dev->features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001527 *
1528 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07001529 *
1530 * It may return NULL if the skb requires no segmentation. This is
1531 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001532 */
Herbert Xu576a30e2006-06-27 13:22:38 -07001533struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001534{
1535 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1536 struct packet_type *ptype;
Al Viro252e33462006-11-14 20:48:11 -08001537 __be16 type = skb->protocol;
Herbert Xua430a432006-07-08 13:34:56 -07001538 int err;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001539
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001540 skb_reset_mac_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001541 skb->mac_len = skb->network_header - skb->mac_header;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001542 __skb_pull(skb, skb->mac_len);
1543
Herbert Xu67fd1a72009-01-19 16:26:44 -08001544 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1545 struct net_device *dev = skb->dev;
1546 struct ethtool_drvinfo info = {};
1547
1548 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1549 dev->ethtool_ops->get_drvinfo(dev, &info);
1550
1551 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d "
1552 "ip_summed=%d",
1553 info.driver, dev ? dev->features : 0L,
1554 skb->sk ? skb->sk->sk_route_caps : 0L,
1555 skb->len, skb->data_len, skb->ip_summed);
1556
Herbert Xua430a432006-07-08 13:34:56 -07001557 if (skb_header_cloned(skb) &&
1558 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1559 return ERR_PTR(err);
1560 }
1561
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001562 rcu_read_lock();
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08001563 list_for_each_entry_rcu(ptype,
1564 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001565 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07001566 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Herbert Xua430a432006-07-08 13:34:56 -07001567 err = ptype->gso_send_check(skb);
1568 segs = ERR_PTR(err);
1569 if (err || skb_gso_ok(skb, features))
1570 break;
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001571 __skb_push(skb, (skb->data -
1572 skb_network_header(skb)));
Herbert Xua430a432006-07-08 13:34:56 -07001573 }
Herbert Xu576a30e2006-06-27 13:22:38 -07001574 segs = ptype->gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001575 break;
1576 }
1577 }
1578 rcu_read_unlock();
1579
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001580 __skb_push(skb, skb->data - skb_mac_header(skb));
Herbert Xu576a30e2006-06-27 13:22:38 -07001581
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001582 return segs;
1583}
1584
1585EXPORT_SYMBOL(skb_gso_segment);
1586
Herbert Xufb286bb2005-11-10 13:01:24 -08001587/* Take action when hardware reception checksum errors are detected. */
1588#ifdef CONFIG_BUG
1589void netdev_rx_csum_fault(struct net_device *dev)
1590{
1591 if (net_ratelimit()) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001592 printk(KERN_ERR "%s: hw csum failure.\n",
Stephen Hemminger246a4212005-12-08 15:21:39 -08001593 dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08001594 dump_stack();
1595 }
1596}
1597EXPORT_SYMBOL(netdev_rx_csum_fault);
1598#endif
1599
Linus Torvalds1da177e2005-04-16 15:20:36 -07001600/* Actually, we should eliminate this check as soon as we know, that:
1601 * 1. IOMMU is present and allows to map all the memory.
1602 * 2. No high memory really exists on this machine.
1603 */
1604
1605static inline int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
1606{
Herbert Xu3d3a8532006-06-27 13:33:10 -07001607#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608 int i;
1609
1610 if (dev->features & NETIF_F_HIGHDMA)
1611 return 0;
1612
1613 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1614 if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1615 return 1;
1616
Herbert Xu3d3a8532006-06-27 13:33:10 -07001617#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618 return 0;
1619}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001620
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001621struct dev_gso_cb {
1622 void (*destructor)(struct sk_buff *skb);
1623};
1624
1625#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1626
1627static void dev_gso_skb_destructor(struct sk_buff *skb)
1628{
1629 struct dev_gso_cb *cb;
1630
1631 do {
1632 struct sk_buff *nskb = skb->next;
1633
1634 skb->next = nskb->next;
1635 nskb->next = NULL;
1636 kfree_skb(nskb);
1637 } while (skb->next);
1638
1639 cb = DEV_GSO_CB(skb);
1640 if (cb->destructor)
1641 cb->destructor(skb);
1642}
1643
1644/**
1645 * dev_gso_segment - Perform emulated hardware segmentation on skb.
1646 * @skb: buffer to segment
1647 *
1648 * This function segments the given skb and stores the list of segments
1649 * in skb->next.
1650 */
1651static int dev_gso_segment(struct sk_buff *skb)
1652{
1653 struct net_device *dev = skb->dev;
1654 struct sk_buff *segs;
Herbert Xu576a30e2006-06-27 13:22:38 -07001655 int features = dev->features & ~(illegal_highdma(dev, skb) ?
1656 NETIF_F_SG : 0);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001657
Herbert Xu576a30e2006-06-27 13:22:38 -07001658 segs = skb_gso_segment(skb, features);
1659
1660 /* Verifying header integrity only. */
1661 if (!segs)
1662 return 0;
1663
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07001664 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001665 return PTR_ERR(segs);
1666
1667 skb->next = segs;
1668 DEV_GSO_CB(skb)->destructor = skb->destructor;
1669 skb->destructor = dev_gso_skb_destructor;
1670
1671 return 0;
1672}
1673
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001674int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
1675 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001676{
Stephen Hemminger00829822008-11-20 20:14:53 -08001677 const struct net_device_ops *ops = dev->netdev_ops;
Patrick Ohlyac45f602009-02-12 05:03:37 +00001678 int rc;
Stephen Hemminger00829822008-11-20 20:14:53 -08001679
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001680 if (likely(!skb->next)) {
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -07001681 if (!list_empty(&ptype_all))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001682 dev_queue_xmit_nit(skb, dev);
1683
Herbert Xu576a30e2006-06-27 13:22:38 -07001684 if (netif_needs_gso(dev, skb)) {
1685 if (unlikely(dev_gso_segment(skb)))
1686 goto out_kfree_skb;
1687 if (skb->next)
1688 goto gso;
1689 }
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001690
Patrick Ohlyac45f602009-02-12 05:03:37 +00001691 rc = ops->ndo_start_xmit(skb, dev);
1692 /*
1693 * TODO: if skb_orphan() was called by
1694 * dev->hard_start_xmit() (for example, the unmodified
1695 * igb driver does that; bnx2 doesn't), then
1696 * skb_tx_software_timestamp() will be unable to send
1697 * back the time stamp.
1698 *
1699 * How can this be prevented? Always create another
1700 * reference to the socket before calling
1701 * dev->hard_start_xmit()? Prevent that skb_orphan()
1702 * does anything in dev->hard_start_xmit() by clearing
1703 * the skb destructor before the call and restoring it
1704 * afterwards, then doing the skb_orphan() ourselves?
1705 */
Patrick Ohlyac45f602009-02-12 05:03:37 +00001706 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001707 }
1708
Herbert Xu576a30e2006-06-27 13:22:38 -07001709gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001710 do {
1711 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001712
1713 skb->next = nskb->next;
1714 nskb->next = NULL;
Stephen Hemminger00829822008-11-20 20:14:53 -08001715 rc = ops->ndo_start_xmit(nskb, dev);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001716 if (unlikely(rc)) {
Michael Chanf54d9e82006-06-25 23:57:04 -07001717 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001718 skb->next = nskb;
1719 return rc;
1720 }
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001721 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07001722 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001723 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001724
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001725 skb->destructor = DEV_GSO_CB(skb)->destructor;
1726
1727out_kfree_skb:
1728 kfree_skb(skb);
1729 return 0;
1730}
1731
David S. Miller70192982009-01-27 16:34:47 -08001732static u32 skb_tx_hashrnd;
David S. Millerb6b2fed2008-07-21 09:48:06 -07001733
Stephen Hemminger92477442009-03-21 13:39:26 -07001734u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)
David S. Miller8f0f2222008-07-15 03:47:03 -07001735{
David S. Miller70192982009-01-27 16:34:47 -08001736 u32 hash;
David S. Millerb6b2fed2008-07-21 09:48:06 -07001737
Eric Dumazetec581f62009-05-01 09:05:06 -07001738 if (skb_rx_queue_recorded(skb))
1739 return skb_get_rx_queue(skb) % dev->real_num_tx_queues;
1740
1741 if (skb->sk && skb->sk->sk_hash)
David S. Miller70192982009-01-27 16:34:47 -08001742 hash = skb->sk->sk_hash;
Eric Dumazetec581f62009-05-01 09:05:06 -07001743 else
David S. Miller70192982009-01-27 16:34:47 -08001744 hash = skb->protocol;
David S. Millerd5a9e242009-01-27 16:22:11 -08001745
David S. Miller70192982009-01-27 16:34:47 -08001746 hash = jhash_1word(hash, skb_tx_hashrnd);
David S. Millerd5a9e242009-01-27 16:22:11 -08001747
David S. Millerb6b2fed2008-07-21 09:48:06 -07001748 return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
David S. Miller8f0f2222008-07-15 03:47:03 -07001749}
Stephen Hemminger92477442009-03-21 13:39:26 -07001750EXPORT_SYMBOL(skb_tx_hash);
David S. Miller8f0f2222008-07-15 03:47:03 -07001751
David S. Millere8a04642008-07-17 00:34:19 -07001752static struct netdev_queue *dev_pick_tx(struct net_device *dev,
1753 struct sk_buff *skb)
1754{
Stephen Hemminger00829822008-11-20 20:14:53 -08001755 const struct net_device_ops *ops = dev->netdev_ops;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001756 u16 queue_index = 0;
1757
Stephen Hemminger00829822008-11-20 20:14:53 -08001758 if (ops->ndo_select_queue)
1759 queue_index = ops->ndo_select_queue(dev, skb);
David S. Miller8f0f2222008-07-15 03:47:03 -07001760 else if (dev->real_num_tx_queues > 1)
David S. Miller70192982009-01-27 16:34:47 -08001761 queue_index = skb_tx_hash(dev, skb);
David S. Millereae792b2008-07-15 03:03:33 -07001762
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001763 skb_set_queue_mapping(skb, queue_index);
1764 return netdev_get_tx_queue(dev, queue_index);
David S. Millere8a04642008-07-17 00:34:19 -07001765}
1766
Dave Jonesd29f7492008-07-22 14:09:06 -07001767/**
1768 * dev_queue_xmit - transmit a buffer
1769 * @skb: buffer to transmit
1770 *
1771 * Queue a buffer for transmission to a network device. The caller must
1772 * have set the device and priority and built the buffer before calling
1773 * this function. The function can be called from an interrupt.
1774 *
1775 * A negative errno code is returned on a failure. A success does not
1776 * guarantee the frame will be transmitted as it may be dropped due
1777 * to congestion or traffic shaping.
1778 *
1779 * -----------------------------------------------------------------------------------
1780 * I notice this method can also return errors from the queue disciplines,
1781 * including NET_XMIT_DROP, which is a positive value. So, errors can also
1782 * be positive.
1783 *
1784 * Regardless of the return value, the skb is consumed, so it is currently
1785 * difficult to retry a send to this method. (You can bump the ref count
1786 * before sending to hold a reference for retry if you are careful.)
1787 *
1788 * When calling this method, interrupts MUST be enabled. This is because
1789 * the BH enable code must have IRQs enabled so that it will not deadlock.
1790 * --BLG
1791 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001792int dev_queue_xmit(struct sk_buff *skb)
1793{
1794 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07001795 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796 struct Qdisc *q;
1797 int rc = -ENOMEM;
1798
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001799 /* GSO will handle the following emulations directly. */
1800 if (netif_needs_gso(dev, skb))
1801 goto gso;
1802
Linus Torvalds1da177e2005-04-16 15:20:36 -07001803 if (skb_shinfo(skb)->frag_list &&
1804 !(dev->features & NETIF_F_FRAGLIST) &&
Herbert Xu364c6ba2006-06-09 16:10:40 -07001805 __skb_linearize(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001806 goto out_kfree_skb;
1807
1808 /* Fragmented skb is linearized if device does not support SG,
1809 * or if at least one of fragments is in highmem and device
1810 * does not support DMA from it.
1811 */
1812 if (skb_shinfo(skb)->nr_frags &&
1813 (!(dev->features & NETIF_F_SG) || illegal_highdma(dev, skb)) &&
Herbert Xu364c6ba2006-06-09 16:10:40 -07001814 __skb_linearize(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815 goto out_kfree_skb;
1816
1817 /* If packet is not checksummed and device does not support
1818 * checksumming for this protocol, complete checksumming here.
1819 */
Herbert Xu663ead32007-04-09 11:59:07 -07001820 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1821 skb_set_transport_header(skb, skb->csum_start -
1822 skb_headroom(skb));
Ben Hutchings6de329e2008-06-16 17:02:28 -07001823 if (!dev_can_checksum(dev, skb) && skb_checksum_help(skb))
1824 goto out_kfree_skb;
Herbert Xu663ead32007-04-09 11:59:07 -07001825 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001826
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001827gso:
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001828 /* Disable soft irqs for various locks below. Also
1829 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001831 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001832
David S. Millereae792b2008-07-15 03:03:33 -07001833 txq = dev_pick_tx(dev, skb);
David S. Millerb0e1e642008-07-08 17:42:10 -07001834 q = rcu_dereference(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07001835
Linus Torvalds1da177e2005-04-16 15:20:36 -07001836#ifdef CONFIG_NET_CLS_ACT
1837 skb->tc_verd = SET_TC_AT(skb->tc_verd,AT_EGRESS);
1838#endif
1839 if (q->enqueue) {
David S. Miller5fb66222008-08-02 20:02:43 -07001840 spinlock_t *root_lock = qdisc_lock(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001841
David S. Miller37437bb2008-07-16 02:15:04 -07001842 spin_lock(root_lock);
1843
David S. Millera9312ae2008-08-17 21:51:03 -07001844 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
David S. Miller96d20312008-08-17 23:37:16 -07001845 kfree_skb(skb);
David S. Millera9312ae2008-08-17 21:51:03 -07001846 rc = NET_XMIT_DROP;
David S. Miller96d20312008-08-17 23:37:16 -07001847 } else {
1848 rc = qdisc_enqueue_root(skb, q);
1849 qdisc_run(q);
David S. Millera9312ae2008-08-17 21:51:03 -07001850 }
David S. Miller37437bb2008-07-16 02:15:04 -07001851 spin_unlock(root_lock);
1852
David S. Miller37437bb2008-07-16 02:15:04 -07001853 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854 }
1855
1856 /* The device has no queue. Common case for software devices:
1857 loopback, all the sorts of tunnels...
1858
Herbert Xu932ff272006-06-09 12:20:56 -07001859 Really, it is unlikely that netif_tx_lock protection is necessary
1860 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07001861 counters.)
1862 However, it is possible, that they rely on protection
1863 made by us here.
1864
1865 Check this and shot the lock. It is not prone from deadlocks.
1866 Either shot noqueue qdisc, it is even simpler 8)
1867 */
1868 if (dev->flags & IFF_UP) {
1869 int cpu = smp_processor_id(); /* ok because BHs are off */
1870
David S. Millerc773e842008-07-08 23:13:53 -07001871 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872
David S. Millerc773e842008-07-08 23:13:53 -07001873 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001875 if (!netif_tx_queue_stopped(txq)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001876 rc = 0;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001877 if (!dev_hard_start_xmit(skb, dev, txq)) {
David S. Millerc773e842008-07-08 23:13:53 -07001878 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001879 goto out;
1880 }
1881 }
David S. Millerc773e842008-07-08 23:13:53 -07001882 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001883 if (net_ratelimit())
1884 printk(KERN_CRIT "Virtual device %s asks to "
1885 "queue packet!\n", dev->name);
1886 } else {
1887 /* Recursion is detected! It is possible,
1888 * unfortunately */
1889 if (net_ratelimit())
1890 printk(KERN_CRIT "Dead loop on virtual device "
1891 "%s, fix it urgently!\n", dev->name);
1892 }
1893 }
1894
1895 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07001896 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897
1898out_kfree_skb:
1899 kfree_skb(skb);
1900 return rc;
1901out:
Herbert Xud4828d82006-06-22 02:28:18 -07001902 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001903 return rc;
1904}
1905
1906
1907/*=======================================================================
1908 Receiver routines
1909 =======================================================================*/
1910
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07001911int netdev_max_backlog __read_mostly = 1000;
1912int netdev_budget __read_mostly = 300;
1913int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001914
1915DEFINE_PER_CPU(struct netif_rx_stats, netdev_rx_stat) = { 0, };
1916
1917
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918/**
1919 * netif_rx - post buffer to the network code
1920 * @skb: buffer to post
1921 *
1922 * This function receives a packet from a device driver and queues it for
1923 * the upper (protocol) levels to process. It always succeeds. The buffer
1924 * may be dropped during processing for congestion control or by the
1925 * protocol layers.
1926 *
1927 * return values:
1928 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001929 * NET_RX_DROP (packet was dropped)
1930 *
1931 */
1932
1933int netif_rx(struct sk_buff *skb)
1934{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001935 struct softnet_data *queue;
1936 unsigned long flags;
1937
1938 /* if netpoll wants it, pretend we never saw it */
1939 if (netpoll_rx(skb))
1940 return NET_RX_DROP;
1941
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07001942 if (!skb->tstamp.tv64)
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001943 net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001944
1945 /*
1946 * The code is rearranged so that the path is the most
1947 * short when CPU is congested, but is still operating.
1948 */
1949 local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001950 queue = &__get_cpu_var(softnet_data);
1951
1952 __get_cpu_var(netdev_rx_stat).total++;
1953 if (queue->input_pkt_queue.qlen <= netdev_max_backlog) {
1954 if (queue->input_pkt_queue.qlen) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001955enqueue:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001956 __skb_queue_tail(&queue->input_pkt_queue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 local_irq_restore(flags);
Stephen Hemminger34008d82005-06-23 20:10:00 -07001958 return NET_RX_SUCCESS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001959 }
1960
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001961 napi_schedule(&queue->backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001962 goto enqueue;
1963 }
1964
Linus Torvalds1da177e2005-04-16 15:20:36 -07001965 __get_cpu_var(netdev_rx_stat).dropped++;
1966 local_irq_restore(flags);
1967
1968 kfree_skb(skb);
1969 return NET_RX_DROP;
1970}
1971
1972int netif_rx_ni(struct sk_buff *skb)
1973{
1974 int err;
1975
1976 preempt_disable();
1977 err = netif_rx(skb);
1978 if (local_softirq_pending())
1979 do_softirq();
1980 preempt_enable();
1981
1982 return err;
1983}
1984
1985EXPORT_SYMBOL(netif_rx_ni);
1986
Linus Torvalds1da177e2005-04-16 15:20:36 -07001987static void net_tx_action(struct softirq_action *h)
1988{
1989 struct softnet_data *sd = &__get_cpu_var(softnet_data);
1990
1991 if (sd->completion_queue) {
1992 struct sk_buff *clist;
1993
1994 local_irq_disable();
1995 clist = sd->completion_queue;
1996 sd->completion_queue = NULL;
1997 local_irq_enable();
1998
1999 while (clist) {
2000 struct sk_buff *skb = clist;
2001 clist = clist->next;
2002
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002003 WARN_ON(atomic_read(&skb->users));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002004 __kfree_skb(skb);
2005 }
2006 }
2007
2008 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07002009 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002010
2011 local_irq_disable();
2012 head = sd->output_queue;
2013 sd->output_queue = NULL;
2014 local_irq_enable();
2015
2016 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07002017 struct Qdisc *q = head;
2018 spinlock_t *root_lock;
2019
Linus Torvalds1da177e2005-04-16 15:20:36 -07002020 head = head->next_sched;
2021
David S. Miller5fb66222008-08-02 20:02:43 -07002022 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07002023 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002024 smp_mb__before_clear_bit();
2025 clear_bit(__QDISC_STATE_SCHED,
2026 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07002027 qdisc_run(q);
2028 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029 } else {
David S. Miller195648b2008-08-19 04:00:36 -07002030 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07002031 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07002032 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07002033 } else {
2034 smp_mb__before_clear_bit();
2035 clear_bit(__QDISC_STATE_SCHED,
2036 &q->state);
2037 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002038 }
2039 }
2040 }
2041}
2042
Stephen Hemminger6f05f622007-03-08 20:46:03 -08002043static inline int deliver_skb(struct sk_buff *skb,
2044 struct packet_type *pt_prev,
2045 struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002046{
2047 atomic_inc(&skb->users);
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002048 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002049}
2050
2051#if defined(CONFIG_BRIDGE) || defined (CONFIG_BRIDGE_MODULE)
Stephen Hemminger6229e362007-03-21 13:38:47 -07002052/* These hooks defined here for ATM */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002053struct net_bridge;
2054struct net_bridge_fdb_entry *(*br_fdb_get_hook)(struct net_bridge *br,
2055 unsigned char *addr);
Stephen Hemminger6229e362007-03-21 13:38:47 -07002056void (*br_fdb_put_hook)(struct net_bridge_fdb_entry *ent) __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002057
Stephen Hemminger6229e362007-03-21 13:38:47 -07002058/*
2059 * If bridge module is loaded call bridging hook.
2060 * returns NULL if packet was consumed.
2061 */
2062struct sk_buff *(*br_handle_frame_hook)(struct net_bridge_port *p,
2063 struct sk_buff *skb) __read_mostly;
2064static inline struct sk_buff *handle_bridge(struct sk_buff *skb,
2065 struct packet_type **pt_prev, int *ret,
2066 struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002067{
2068 struct net_bridge_port *port;
2069
Stephen Hemminger6229e362007-03-21 13:38:47 -07002070 if (skb->pkt_type == PACKET_LOOPBACK ||
2071 (port = rcu_dereference(skb->dev->br_port)) == NULL)
2072 return skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002073
2074 if (*pt_prev) {
Stephen Hemminger6229e362007-03-21 13:38:47 -07002075 *ret = deliver_skb(skb, *pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002076 *pt_prev = NULL;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002077 }
2078
Stephen Hemminger6229e362007-03-21 13:38:47 -07002079 return br_handle_frame_hook(port, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080}
2081#else
Stephen Hemminger6229e362007-03-21 13:38:47 -07002082#define handle_bridge(skb, pt_prev, ret, orig_dev) (skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002083#endif
2084
Patrick McHardyb863ceb2007-07-14 18:55:06 -07002085#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE)
2086struct sk_buff *(*macvlan_handle_frame_hook)(struct sk_buff *skb) __read_mostly;
2087EXPORT_SYMBOL_GPL(macvlan_handle_frame_hook);
2088
2089static inline struct sk_buff *handle_macvlan(struct sk_buff *skb,
2090 struct packet_type **pt_prev,
2091 int *ret,
2092 struct net_device *orig_dev)
2093{
2094 if (skb->dev->macvlan_port == NULL)
2095 return skb;
2096
2097 if (*pt_prev) {
2098 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2099 *pt_prev = NULL;
2100 }
2101 return macvlan_handle_frame_hook(skb);
2102}
2103#else
2104#define handle_macvlan(skb, pt_prev, ret, orig_dev) (skb)
2105#endif
2106
Linus Torvalds1da177e2005-04-16 15:20:36 -07002107#ifdef CONFIG_NET_CLS_ACT
2108/* TODO: Maybe we should just force sch_ingress to be compiled in
2109 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
2110 * a compare and 2 stores extra right now if we dont have it on
2111 * but have CONFIG_NET_CLS_ACT
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002112 * NOTE: This doesnt stop any functionality; if you dont have
Linus Torvalds1da177e2005-04-16 15:20:36 -07002113 * the ingress scheduler, you just cant add policies on ingress.
2114 *
2115 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002116static int ing_filter(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002117{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002118 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07002119 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07002120 struct netdev_queue *rxq;
2121 int result = TC_ACT_OK;
2122 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002123
Herbert Xuf697c3e2007-10-14 00:38:47 -07002124 if (MAX_RED_LOOP < ttl++) {
2125 printk(KERN_WARNING
2126 "Redir loop detected Dropping packet (%d->%d)\n",
2127 skb->iif, dev->ifindex);
2128 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002129 }
2130
Herbert Xuf697c3e2007-10-14 00:38:47 -07002131 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
2132 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
2133
David S. Miller555353c2008-07-08 17:33:13 -07002134 rxq = &dev->rx_queue;
2135
David S. Miller83874002008-07-17 00:53:03 -07002136 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07002137 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07002138 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07002139 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
2140 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07002141 spin_unlock(qdisc_lock(q));
2142 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07002143
Linus Torvalds1da177e2005-04-16 15:20:36 -07002144 return result;
2145}
Herbert Xuf697c3e2007-10-14 00:38:47 -07002146
2147static inline struct sk_buff *handle_ing(struct sk_buff *skb,
2148 struct packet_type **pt_prev,
2149 int *ret, struct net_device *orig_dev)
2150{
David S. Miller8d50b532008-07-30 02:37:46 -07002151 if (skb->dev->rx_queue.qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07002152 goto out;
2153
2154 if (*pt_prev) {
2155 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2156 *pt_prev = NULL;
2157 } else {
2158 /* Huh? Why does turning on AF_PACKET affect this? */
2159 skb->tc_verd = SET_TC_OK2MUNGE(skb->tc_verd);
2160 }
2161
2162 switch (ing_filter(skb)) {
2163 case TC_ACT_SHOT:
2164 case TC_ACT_STOLEN:
2165 kfree_skb(skb);
2166 return NULL;
2167 }
2168
2169out:
2170 skb->tc_verd = 0;
2171 return skb;
2172}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002173#endif
2174
Patrick McHardybc1d0412008-07-14 22:49:30 -07002175/*
2176 * netif_nit_deliver - deliver received packets to network taps
2177 * @skb: buffer
2178 *
2179 * This function is used to deliver incoming packets to network
2180 * taps. It should be used when the normal netif_receive_skb path
2181 * is bypassed, for example because of VLAN acceleration.
2182 */
2183void netif_nit_deliver(struct sk_buff *skb)
2184{
2185 struct packet_type *ptype;
2186
2187 if (list_empty(&ptype_all))
2188 return;
2189
2190 skb_reset_network_header(skb);
2191 skb_reset_transport_header(skb);
2192 skb->mac_len = skb->network_header - skb->mac_header;
2193
2194 rcu_read_lock();
2195 list_for_each_entry_rcu(ptype, &ptype_all, list) {
2196 if (!ptype->dev || ptype->dev == skb->dev)
2197 deliver_skb(skb, ptype, skb->dev);
2198 }
2199 rcu_read_unlock();
2200}
2201
Stephen Hemminger3b582cc2007-11-01 02:21:47 -07002202/**
2203 * netif_receive_skb - process receive buffer from network
2204 * @skb: buffer to process
2205 *
2206 * netif_receive_skb() is the main receive data processing function.
2207 * It always succeeds. The buffer may be dropped during processing
2208 * for congestion control or by the protocol layers.
2209 *
2210 * This function may only be called from softirq context and interrupts
2211 * should be enabled.
2212 *
2213 * Return values (usually ignored):
2214 * NET_RX_SUCCESS: no congestion
2215 * NET_RX_DROP: packet was dropped
2216 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002217int netif_receive_skb(struct sk_buff *skb)
2218{
2219 struct packet_type *ptype, *pt_prev;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002220 struct net_device *orig_dev;
Joe Eykholt0d7a3682008-07-02 18:22:01 -07002221 struct net_device *null_or_orig;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002222 int ret = NET_RX_DROP;
Al Viro252e33462006-11-14 20:48:11 -08002223 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002224
Patrick McHardy9b22ea52008-11-04 14:49:57 -08002225 if (skb->vlan_tci && vlan_hwaccel_do_receive(skb))
2226 return NET_RX_SUCCESS;
2227
Linus Torvalds1da177e2005-04-16 15:20:36 -07002228 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002229 if (netpoll_receive_skb(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002230 return NET_RX_DROP;
2231
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07002232 if (!skb->tstamp.tv64)
Patrick McHardya61bbcf2005-08-14 17:24:31 -07002233 net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002234
Patrick McHardyc01003c2007-03-29 11:46:52 -07002235 if (!skb->iif)
2236 skb->iif = skb->dev->ifindex;
David S. Miller86e65da2005-08-09 19:36:29 -07002237
Joe Eykholt0d7a3682008-07-02 18:22:01 -07002238 null_or_orig = NULL;
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07002239 orig_dev = skb->dev;
2240 if (orig_dev->master) {
Joe Eykholt0d7a3682008-07-02 18:22:01 -07002241 if (skb_bond_should_drop(skb))
2242 null_or_orig = orig_dev; /* deliver only exact match */
2243 else
2244 skb->dev = orig_dev->master;
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07002245 }
Jay Vosburgh8f903c72006-02-21 16:36:44 -08002246
Linus Torvalds1da177e2005-04-16 15:20:36 -07002247 __get_cpu_var(netdev_rx_stat).total++;
2248
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07002249 skb_reset_network_header(skb);
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -03002250 skb_reset_transport_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07002251 skb->mac_len = skb->network_header - skb->mac_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002252
2253 pt_prev = NULL;
2254
2255 rcu_read_lock();
2256
2257#ifdef CONFIG_NET_CLS_ACT
2258 if (skb->tc_verd & TC_NCLS) {
2259 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
2260 goto ncls;
2261 }
2262#endif
2263
2264 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Joe Eykholtf9823072008-07-02 18:22:02 -07002265 if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
2266 ptype->dev == orig_dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002267 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002268 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002269 pt_prev = ptype;
2270 }
2271 }
2272
2273#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07002274 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
2275 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002276 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002277ncls:
2278#endif
2279
Stephen Hemminger6229e362007-03-21 13:38:47 -07002280 skb = handle_bridge(skb, &pt_prev, &ret, orig_dev);
2281 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002282 goto out;
Patrick McHardyb863ceb2007-07-14 18:55:06 -07002283 skb = handle_macvlan(skb, &pt_prev, &ret, orig_dev);
2284 if (!skb)
2285 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002286
Herbert Xu9a279bc2009-02-04 16:55:27 -08002287 skb_orphan(skb);
2288
Linus Torvalds1da177e2005-04-16 15:20:36 -07002289 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08002290 list_for_each_entry_rcu(ptype,
2291 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002292 if (ptype->type == type &&
Joe Eykholtf9823072008-07-02 18:22:02 -07002293 (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
2294 ptype->dev == orig_dev)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002295 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002296 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002297 pt_prev = ptype;
2298 }
2299 }
2300
2301 if (pt_prev) {
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002302 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002303 } else {
2304 kfree_skb(skb);
2305 /* Jamal, now you will not able to escape explaining
2306 * me how you were going to use this. :-)
2307 */
2308 ret = NET_RX_DROP;
2309 }
2310
2311out:
2312 rcu_read_unlock();
2313 return ret;
2314}
2315
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07002316/* Network device is going away, flush any packets still pending */
2317static void flush_backlog(void *arg)
2318{
2319 struct net_device *dev = arg;
2320 struct softnet_data *queue = &__get_cpu_var(softnet_data);
2321 struct sk_buff *skb, *tmp;
2322
2323 skb_queue_walk_safe(&queue->input_pkt_queue, skb, tmp)
2324 if (skb->dev == dev) {
2325 __skb_unlink(skb, &queue->input_pkt_queue);
2326 kfree_skb(skb);
2327 }
2328}
2329
Herbert Xud565b0a2008-12-15 23:38:52 -08002330static int napi_gro_complete(struct sk_buff *skb)
2331{
2332 struct packet_type *ptype;
2333 __be16 type = skb->protocol;
2334 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
2335 int err = -ENOENT;
2336
Herbert Xufc59f9a2009-04-14 15:11:06 -07002337 if (NAPI_GRO_CB(skb)->count == 1) {
2338 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08002339 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07002340 }
Herbert Xud565b0a2008-12-15 23:38:52 -08002341
2342 rcu_read_lock();
2343 list_for_each_entry_rcu(ptype, head, list) {
2344 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
2345 continue;
2346
2347 err = ptype->gro_complete(skb);
2348 break;
2349 }
2350 rcu_read_unlock();
2351
2352 if (err) {
2353 WARN_ON(&ptype->list == head);
2354 kfree_skb(skb);
2355 return NET_RX_SUCCESS;
2356 }
2357
2358out:
Herbert Xud565b0a2008-12-15 23:38:52 -08002359 return netif_receive_skb(skb);
2360}
2361
2362void napi_gro_flush(struct napi_struct *napi)
2363{
2364 struct sk_buff *skb, *next;
2365
2366 for (skb = napi->gro_list; skb; skb = next) {
2367 next = skb->next;
2368 skb->next = NULL;
2369 napi_gro_complete(skb);
2370 }
2371
Herbert Xu4ae55442009-02-08 18:00:36 +00002372 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08002373 napi->gro_list = NULL;
2374}
2375EXPORT_SYMBOL(napi_gro_flush);
2376
Herbert Xu86911732009-01-29 14:19:50 +00002377void *skb_gro_header(struct sk_buff *skb, unsigned int hlen)
2378{
2379 unsigned int offset = skb_gro_offset(skb);
2380
2381 hlen += offset;
Herbert Xuedbd9e32009-04-27 05:44:29 -07002382 if (unlikely(skb_headlen(skb) ||
2383 skb_shinfo(skb)->frags[0].size < hlen ||
Herbert Xu86911732009-01-29 14:19:50 +00002384 PageHighMem(skb_shinfo(skb)->frags[0].page)))
2385 return pskb_may_pull(skb, hlen) ? skb->data + offset : NULL;
2386
2387 return page_address(skb_shinfo(skb)->frags[0].page) +
Herbert Xuedbd9e32009-04-27 05:44:29 -07002388 skb_shinfo(skb)->frags[0].page_offset + offset;
Herbert Xu86911732009-01-29 14:19:50 +00002389}
2390EXPORT_SYMBOL(skb_gro_header);
2391
Herbert Xu96e93ea2009-01-06 10:49:34 -08002392int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08002393{
2394 struct sk_buff **pp = NULL;
2395 struct packet_type *ptype;
2396 __be16 type = skb->protocol;
2397 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
Herbert Xu0da2afd52008-12-26 14:57:42 -08002398 int same_flow;
Herbert Xud565b0a2008-12-15 23:38:52 -08002399 int mac_len;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002400 int ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08002401
2402 if (!(skb->dev->features & NETIF_F_GRO))
2403 goto normal;
2404
Herbert Xuf17f5c92009-01-14 14:36:12 -08002405 if (skb_is_gso(skb) || skb_shinfo(skb)->frag_list)
2406 goto normal;
2407
Herbert Xud565b0a2008-12-15 23:38:52 -08002408 rcu_read_lock();
2409 list_for_each_entry_rcu(ptype, head, list) {
Herbert Xud565b0a2008-12-15 23:38:52 -08002410 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
2411 continue;
2412
Herbert Xu86911732009-01-29 14:19:50 +00002413 skb_set_network_header(skb, skb_gro_offset(skb));
Herbert Xud565b0a2008-12-15 23:38:52 -08002414 mac_len = skb->network_header - skb->mac_header;
2415 skb->mac_len = mac_len;
2416 NAPI_GRO_CB(skb)->same_flow = 0;
2417 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08002418 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08002419
Herbert Xud565b0a2008-12-15 23:38:52 -08002420 pp = ptype->gro_receive(&napi->gro_list, skb);
2421 break;
2422 }
2423 rcu_read_unlock();
2424
2425 if (&ptype->list == head)
2426 goto normal;
2427
Herbert Xu0da2afd52008-12-26 14:57:42 -08002428 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002429 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08002430
Herbert Xud565b0a2008-12-15 23:38:52 -08002431 if (pp) {
2432 struct sk_buff *nskb = *pp;
2433
2434 *pp = nskb->next;
2435 nskb->next = NULL;
2436 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00002437 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08002438 }
2439
Herbert Xu0da2afd52008-12-26 14:57:42 -08002440 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08002441 goto ok;
2442
Herbert Xu4ae55442009-02-08 18:00:36 +00002443 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08002444 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08002445
Herbert Xu4ae55442009-02-08 18:00:36 +00002446 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08002447 NAPI_GRO_CB(skb)->count = 1;
Herbert Xu86911732009-01-29 14:19:50 +00002448 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08002449 skb->next = napi->gro_list;
2450 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002451 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08002452
Herbert Xuad0f9902009-02-01 01:24:55 -08002453pull:
2454 if (unlikely(!pskb_may_pull(skb, skb_gro_offset(skb)))) {
2455 if (napi->gro_list == skb)
2456 napi->gro_list = skb->next;
2457 ret = GRO_DROP;
2458 }
2459
Herbert Xud565b0a2008-12-15 23:38:52 -08002460ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002461 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08002462
2463normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08002464 ret = GRO_NORMAL;
2465 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08002466}
Herbert Xu96e93ea2009-01-06 10:49:34 -08002467EXPORT_SYMBOL(dev_gro_receive);
2468
2469static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2470{
2471 struct sk_buff *p;
2472
Herbert Xud1c76af2009-03-16 10:50:02 -07002473 if (netpoll_rx_on(skb))
2474 return GRO_NORMAL;
2475
Herbert Xu96e93ea2009-01-06 10:49:34 -08002476 for (p = napi->gro_list; p; p = p->next) {
Stephen Hemmingerf2bde732009-04-01 11:20:20 +00002477 NAPI_GRO_CB(p)->same_flow = (p->dev == skb->dev)
2478 && !compare_ether_header(skb_mac_header(p),
2479 skb_gro_mac_header(skb));
Herbert Xu96e93ea2009-01-06 10:49:34 -08002480 NAPI_GRO_CB(p)->flush = 0;
2481 }
2482
2483 return dev_gro_receive(napi, skb);
2484}
Herbert Xu5d38a072009-01-04 16:13:40 -08002485
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002486int napi_skb_finish(int ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08002487{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002488 int err = NET_RX_SUCCESS;
2489
2490 switch (ret) {
2491 case GRO_NORMAL:
Herbert Xu5d38a072009-01-04 16:13:40 -08002492 return netif_receive_skb(skb);
2493
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002494 case GRO_DROP:
2495 err = NET_RX_DROP;
2496 /* fall through */
2497
2498 case GRO_MERGED_FREE:
Herbert Xu5d38a072009-01-04 16:13:40 -08002499 kfree_skb(skb);
2500 break;
2501 }
2502
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002503 return err;
2504}
2505EXPORT_SYMBOL(napi_skb_finish);
2506
2507int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2508{
Herbert Xu86911732009-01-29 14:19:50 +00002509 skb_gro_reset_offset(skb);
2510
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002511 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08002512}
2513EXPORT_SYMBOL(napi_gro_receive);
2514
Herbert Xu96e93ea2009-01-06 10:49:34 -08002515void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
2516{
Herbert Xu96e93ea2009-01-06 10:49:34 -08002517 __skb_pull(skb, skb_headlen(skb));
2518 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
2519
2520 napi->skb = skb;
2521}
2522EXPORT_SYMBOL(napi_reuse_skb);
2523
Herbert Xu76620aa2009-04-16 02:02:07 -07002524struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08002525{
2526 struct net_device *dev = napi->dev;
2527 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08002528
2529 if (!skb) {
2530 skb = netdev_alloc_skb(dev, GRO_MAX_HEAD + NET_IP_ALIGN);
2531 if (!skb)
2532 goto out;
2533
2534 skb_reserve(skb, NET_IP_ALIGN);
Herbert Xu76620aa2009-04-16 02:02:07 -07002535
2536 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08002537 }
2538
Herbert Xu96e93ea2009-01-06 10:49:34 -08002539out:
2540 return skb;
2541}
Herbert Xu76620aa2009-04-16 02:02:07 -07002542EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08002543
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002544int napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb, int ret)
2545{
2546 int err = NET_RX_SUCCESS;
2547
2548 switch (ret) {
2549 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00002550 case GRO_HELD:
Herbert Xu86911732009-01-29 14:19:50 +00002551 skb->protocol = eth_type_trans(skb, napi->dev);
2552
2553 if (ret == GRO_NORMAL)
2554 return netif_receive_skb(skb);
2555
2556 skb_gro_pull(skb, -ETH_HLEN);
2557 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002558
2559 case GRO_DROP:
2560 err = NET_RX_DROP;
2561 /* fall through */
2562
2563 case GRO_MERGED_FREE:
2564 napi_reuse_skb(napi, skb);
2565 break;
2566 }
2567
2568 return err;
2569}
2570EXPORT_SYMBOL(napi_frags_finish);
2571
Herbert Xu76620aa2009-04-16 02:02:07 -07002572struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08002573{
Herbert Xu76620aa2009-04-16 02:02:07 -07002574 struct sk_buff *skb = napi->skb;
2575 struct ethhdr *eth;
2576
2577 napi->skb = NULL;
2578
2579 skb_reset_mac_header(skb);
2580 skb_gro_reset_offset(skb);
2581
2582 eth = skb_gro_header(skb, sizeof(*eth));
2583 if (!eth) {
2584 napi_reuse_skb(napi, skb);
2585 skb = NULL;
2586 goto out;
2587 }
2588
2589 skb_gro_pull(skb, sizeof(*eth));
2590
2591 /*
2592 * This works because the only protocols we care about don't require
2593 * special handling. We'll fix it up properly at the end.
2594 */
2595 skb->protocol = eth->h_proto;
2596
2597out:
2598 return skb;
2599}
2600EXPORT_SYMBOL(napi_frags_skb);
2601
2602int napi_gro_frags(struct napi_struct *napi)
2603{
2604 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08002605
2606 if (!skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002607 return NET_RX_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08002608
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002609 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08002610}
2611EXPORT_SYMBOL(napi_gro_frags);
2612
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002613static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002614{
2615 int work = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002616 struct softnet_data *queue = &__get_cpu_var(softnet_data);
2617 unsigned long start_time = jiffies;
2618
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002619 napi->weight = weight_p;
2620 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002621 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002622
2623 local_irq_disable();
2624 skb = __skb_dequeue(&queue->input_pkt_queue);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002625 if (!skb) {
Herbert Xu8f1ead22009-03-26 00:59:10 -07002626 __napi_complete(napi);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002627 local_irq_enable();
Herbert Xu8f1ead22009-03-26 00:59:10 -07002628 break;
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002629 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002630 local_irq_enable();
2631
Herbert Xu8f1ead22009-03-26 00:59:10 -07002632 netif_receive_skb(skb);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002633 } while (++work < quota && jiffies == start_time);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002634
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002635 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002636}
2637
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002638/**
2639 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07002640 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002641 *
2642 * The entry's receive function will be scheduled to run
2643 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08002644void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002645{
2646 unsigned long flags;
2647
2648 local_irq_save(flags);
2649 list_add_tail(&n->poll_list, &__get_cpu_var(softnet_data).poll_list);
2650 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2651 local_irq_restore(flags);
2652}
2653EXPORT_SYMBOL(__napi_schedule);
2654
Herbert Xud565b0a2008-12-15 23:38:52 -08002655void __napi_complete(struct napi_struct *n)
2656{
2657 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
2658 BUG_ON(n->gro_list);
2659
2660 list_del(&n->poll_list);
2661 smp_mb__before_clear_bit();
2662 clear_bit(NAPI_STATE_SCHED, &n->state);
2663}
2664EXPORT_SYMBOL(__napi_complete);
2665
2666void napi_complete(struct napi_struct *n)
2667{
2668 unsigned long flags;
2669
2670 /*
2671 * don't let napi dequeue from the cpu poll list
2672 * just in case its running on a different cpu
2673 */
2674 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
2675 return;
2676
2677 napi_gro_flush(n);
2678 local_irq_save(flags);
2679 __napi_complete(n);
2680 local_irq_restore(flags);
2681}
2682EXPORT_SYMBOL(napi_complete);
2683
2684void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2685 int (*poll)(struct napi_struct *, int), int weight)
2686{
2687 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00002688 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08002689 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08002690 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08002691 napi->poll = poll;
2692 napi->weight = weight;
2693 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08002694 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08002695#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08002696 spin_lock_init(&napi->poll_lock);
2697 napi->poll_owner = -1;
2698#endif
2699 set_bit(NAPI_STATE_SCHED, &napi->state);
2700}
2701EXPORT_SYMBOL(netif_napi_add);
2702
2703void netif_napi_del(struct napi_struct *napi)
2704{
2705 struct sk_buff *skb, *next;
2706
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08002707 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07002708 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08002709
2710 for (skb = napi->gro_list; skb; skb = next) {
2711 next = skb->next;
2712 skb->next = NULL;
2713 kfree_skb(skb);
2714 }
2715
2716 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00002717 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08002718}
2719EXPORT_SYMBOL(netif_napi_del);
2720
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002721
Linus Torvalds1da177e2005-04-16 15:20:36 -07002722static void net_rx_action(struct softirq_action *h)
2723{
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002724 struct list_head *list = &__get_cpu_var(softnet_data).poll_list;
Stephen Hemminger24f8b232008-11-03 17:14:38 -08002725 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07002726 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07002727 void *have;
2728
Linus Torvalds1da177e2005-04-16 15:20:36 -07002729 local_irq_disable();
2730
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002731 while (!list_empty(list)) {
2732 struct napi_struct *n;
2733 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002734
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002735 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08002736 * Allow this to run for 2 jiffies since which will allow
2737 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002738 */
Stephen Hemminger24f8b232008-11-03 17:14:38 -08002739 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002740 goto softnet_break;
2741
2742 local_irq_enable();
2743
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002744 /* Even though interrupts have been re-enabled, this
2745 * access is safe because interrupts can only add new
2746 * entries to the tail of this list, and only ->poll()
2747 * calls can remove this head entry from the list.
2748 */
2749 n = list_entry(list->next, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002750
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002751 have = netpoll_poll_lock(n);
2752
2753 weight = n->weight;
2754
David S. Miller0a7606c2007-10-29 21:28:47 -07002755 /* This NAPI_STATE_SCHED test is for avoiding a race
2756 * with netpoll's poll_napi(). Only the entity which
2757 * obtains the lock and sees NAPI_STATE_SCHED set will
2758 * actually make the ->poll() call. Therefore we avoid
2759 * accidently calling ->poll() when NAPI is not scheduled.
2760 */
2761 work = 0;
2762 if (test_bit(NAPI_STATE_SCHED, &n->state))
2763 work = n->poll(n, weight);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002764
2765 WARN_ON_ONCE(work > weight);
2766
2767 budget -= work;
2768
2769 local_irq_disable();
2770
2771 /* Drivers must not modify the NAPI state if they
2772 * consume the entire weight. In such cases this code
2773 * still "owns" the NAPI instance and therefore can
2774 * move the instance around on the list at-will.
2775 */
David S. Millerfed17f32008-01-07 21:00:40 -08002776 if (unlikely(work == weight)) {
2777 if (unlikely(napi_disable_pending(n)))
2778 __napi_complete(n);
2779 else
2780 list_move_tail(&n->poll_list, list);
2781 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002782
2783 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002784 }
2785out:
Shannon Nelson515e06c2007-06-23 23:09:23 -07002786 local_irq_enable();
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002787
Chris Leechdb217332006-06-17 21:24:58 -07002788#ifdef CONFIG_NET_DMA
2789 /*
2790 * There may not be any more sk_buffs coming right now, so push
2791 * any pending DMA copies to hardware
2792 */
Dan Williams2ba05622009-01-06 11:38:14 -07002793 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07002794#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002795
Linus Torvalds1da177e2005-04-16 15:20:36 -07002796 return;
2797
2798softnet_break:
2799 __get_cpu_var(netdev_rx_stat).time_squeeze++;
2800 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2801 goto out;
2802}
2803
2804static gifconf_func_t * gifconf_list [NPROTO];
2805
2806/**
2807 * register_gifconf - register a SIOCGIF handler
2808 * @family: Address family
2809 * @gifconf: Function handler
2810 *
2811 * Register protocol dependent address dumping routines. The handler
2812 * that is passed must not be freed or reused until it has been replaced
2813 * by another handler.
2814 */
2815int register_gifconf(unsigned int family, gifconf_func_t * gifconf)
2816{
2817 if (family >= NPROTO)
2818 return -EINVAL;
2819 gifconf_list[family] = gifconf;
2820 return 0;
2821}
2822
2823
2824/*
2825 * Map an interface index to its name (SIOCGIFNAME)
2826 */
2827
2828/*
2829 * We need this ioctl for efficient implementation of the
2830 * if_indextoname() function required by the IPv6 API. Without
2831 * it, we would have to search all the interfaces to find a
2832 * match. --pb
2833 */
2834
Eric W. Biederman881d9662007-09-17 11:56:21 -07002835static int dev_ifname(struct net *net, struct ifreq __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002836{
2837 struct net_device *dev;
2838 struct ifreq ifr;
2839
2840 /*
2841 * Fetch the caller's info block.
2842 */
2843
2844 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
2845 return -EFAULT;
2846
2847 read_lock(&dev_base_lock);
Eric W. Biederman881d9662007-09-17 11:56:21 -07002848 dev = __dev_get_by_index(net, ifr.ifr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002849 if (!dev) {
2850 read_unlock(&dev_base_lock);
2851 return -ENODEV;
2852 }
2853
2854 strcpy(ifr.ifr_name, dev->name);
2855 read_unlock(&dev_base_lock);
2856
2857 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
2858 return -EFAULT;
2859 return 0;
2860}
2861
2862/*
2863 * Perform a SIOCGIFCONF call. This structure will change
2864 * size eventually, and there is nothing I can do about it.
2865 * Thus we will need a 'compatibility mode'.
2866 */
2867
Eric W. Biederman881d9662007-09-17 11:56:21 -07002868static int dev_ifconf(struct net *net, char __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002869{
2870 struct ifconf ifc;
2871 struct net_device *dev;
2872 char __user *pos;
2873 int len;
2874 int total;
2875 int i;
2876
2877 /*
2878 * Fetch the caller's info block.
2879 */
2880
2881 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
2882 return -EFAULT;
2883
2884 pos = ifc.ifc_buf;
2885 len = ifc.ifc_len;
2886
2887 /*
2888 * Loop over the interfaces, and write an info block for each.
2889 */
2890
2891 total = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07002892 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002893 for (i = 0; i < NPROTO; i++) {
2894 if (gifconf_list[i]) {
2895 int done;
2896 if (!pos)
2897 done = gifconf_list[i](dev, NULL, 0);
2898 else
2899 done = gifconf_list[i](dev, pos + total,
2900 len - total);
2901 if (done < 0)
2902 return -EFAULT;
2903 total += done;
2904 }
2905 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002906 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002907
2908 /*
2909 * All done. Write the updated control block back to the caller.
2910 */
2911 ifc.ifc_len = total;
2912
2913 /*
2914 * Both BSD and Solaris return 0 here, so we do too.
2915 */
2916 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
2917}
2918
2919#ifdef CONFIG_PROC_FS
2920/*
2921 * This is invoked by the /proc filesystem handler to display a device
2922 * in detail.
2923 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002924void *dev_seq_start(struct seq_file *seq, loff_t *pos)
Eric Dumazet9a429c42008-01-01 21:58:02 -08002925 __acquires(dev_base_lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002926{
Denis V. Luneve372c412007-11-19 22:31:54 -08002927 struct net *net = seq_file_net(seq);
Pavel Emelianov7562f872007-05-03 15:13:45 -07002928 loff_t off;
2929 struct net_device *dev;
2930
Linus Torvalds1da177e2005-04-16 15:20:36 -07002931 read_lock(&dev_base_lock);
Pavel Emelianov7562f872007-05-03 15:13:45 -07002932 if (!*pos)
2933 return SEQ_START_TOKEN;
2934
2935 off = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07002936 for_each_netdev(net, dev)
Pavel Emelianov7562f872007-05-03 15:13:45 -07002937 if (off++ == *pos)
2938 return dev;
2939
2940 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002941}
2942
2943void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2944{
Denis V. Luneve372c412007-11-19 22:31:54 -08002945 struct net *net = seq_file_net(seq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002946 ++*pos;
Pavel Emelianov7562f872007-05-03 15:13:45 -07002947 return v == SEQ_START_TOKEN ?
Eric W. Biederman881d9662007-09-17 11:56:21 -07002948 first_net_device(net) : next_net_device((struct net_device *)v);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002949}
2950
2951void dev_seq_stop(struct seq_file *seq, void *v)
Eric Dumazet9a429c42008-01-01 21:58:02 -08002952 __releases(dev_base_lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002953{
2954 read_unlock(&dev_base_lock);
2955}
2956
2957static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
2958{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08002959 const struct net_device_stats *stats = dev_get_stats(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002960
Rusty Russell5a1b5892007-04-28 21:04:03 -07002961 seq_printf(seq, "%6s:%8lu %7lu %4lu %4lu %4lu %5lu %10lu %9lu "
2962 "%8lu %7lu %4lu %4lu %4lu %5lu %7lu %10lu\n",
2963 dev->name, stats->rx_bytes, stats->rx_packets,
2964 stats->rx_errors,
2965 stats->rx_dropped + stats->rx_missed_errors,
2966 stats->rx_fifo_errors,
2967 stats->rx_length_errors + stats->rx_over_errors +
2968 stats->rx_crc_errors + stats->rx_frame_errors,
2969 stats->rx_compressed, stats->multicast,
2970 stats->tx_bytes, stats->tx_packets,
2971 stats->tx_errors, stats->tx_dropped,
2972 stats->tx_fifo_errors, stats->collisions,
2973 stats->tx_carrier_errors +
2974 stats->tx_aborted_errors +
2975 stats->tx_window_errors +
2976 stats->tx_heartbeat_errors,
2977 stats->tx_compressed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002978}
2979
2980/*
2981 * Called from the PROCfs module. This now uses the new arbitrary sized
2982 * /proc/net interface to create /proc/net/dev
2983 */
2984static int dev_seq_show(struct seq_file *seq, void *v)
2985{
2986 if (v == SEQ_START_TOKEN)
2987 seq_puts(seq, "Inter-| Receive "
2988 " | Transmit\n"
2989 " face |bytes packets errs drop fifo frame "
2990 "compressed multicast|bytes packets errs "
2991 "drop fifo colls carrier compressed\n");
2992 else
2993 dev_seq_printf_stats(seq, v);
2994 return 0;
2995}
2996
2997static struct netif_rx_stats *softnet_get_online(loff_t *pos)
2998{
2999 struct netif_rx_stats *rc = NULL;
3000
Mike Travis0c0b0ac2008-05-02 16:43:08 -07003001 while (*pos < nr_cpu_ids)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003002 if (cpu_online(*pos)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003003 rc = &per_cpu(netdev_rx_stat, *pos);
3004 break;
3005 } else
3006 ++*pos;
3007 return rc;
3008}
3009
3010static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
3011{
3012 return softnet_get_online(pos);
3013}
3014
3015static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3016{
3017 ++*pos;
3018 return softnet_get_online(pos);
3019}
3020
3021static void softnet_seq_stop(struct seq_file *seq, void *v)
3022{
3023}
3024
3025static int softnet_seq_show(struct seq_file *seq, void *v)
3026{
3027 struct netif_rx_stats *s = v;
3028
3029 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
Stephen Hemminger31aa02c2005-06-23 20:12:48 -07003030 s->total, s->dropped, s->time_squeeze, 0,
Stephen Hemmingerc1ebcdb2005-06-23 20:08:59 -07003031 0, 0, 0, 0, /* was fastroute */
3032 s->cpu_collision );
Linus Torvalds1da177e2005-04-16 15:20:36 -07003033 return 0;
3034}
3035
Stephen Hemmingerf6908082007-03-12 14:34:29 -07003036static const struct seq_operations dev_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003037 .start = dev_seq_start,
3038 .next = dev_seq_next,
3039 .stop = dev_seq_stop,
3040 .show = dev_seq_show,
3041};
3042
3043static int dev_seq_open(struct inode *inode, struct file *file)
3044{
Denis V. Luneve372c412007-11-19 22:31:54 -08003045 return seq_open_net(inode, file, &dev_seq_ops,
3046 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003047}
3048
Arjan van de Ven9a321442007-02-12 00:55:35 -08003049static const struct file_operations dev_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003050 .owner = THIS_MODULE,
3051 .open = dev_seq_open,
3052 .read = seq_read,
3053 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08003054 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003055};
3056
Stephen Hemmingerf6908082007-03-12 14:34:29 -07003057static const struct seq_operations softnet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058 .start = softnet_seq_start,
3059 .next = softnet_seq_next,
3060 .stop = softnet_seq_stop,
3061 .show = softnet_seq_show,
3062};
3063
3064static int softnet_seq_open(struct inode *inode, struct file *file)
3065{
3066 return seq_open(file, &softnet_seq_ops);
3067}
3068
Arjan van de Ven9a321442007-02-12 00:55:35 -08003069static const struct file_operations softnet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003070 .owner = THIS_MODULE,
3071 .open = softnet_seq_open,
3072 .read = seq_read,
3073 .llseek = seq_lseek,
3074 .release = seq_release,
3075};
3076
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003077static void *ptype_get_idx(loff_t pos)
3078{
3079 struct packet_type *pt = NULL;
3080 loff_t i = 0;
3081 int t;
3082
3083 list_for_each_entry_rcu(pt, &ptype_all, list) {
3084 if (i == pos)
3085 return pt;
3086 ++i;
3087 }
3088
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003089 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003090 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
3091 if (i == pos)
3092 return pt;
3093 ++i;
3094 }
3095 }
3096 return NULL;
3097}
3098
3099static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
Stephen Hemminger72348a42008-01-21 02:27:29 -08003100 __acquires(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003101{
3102 rcu_read_lock();
3103 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
3104}
3105
3106static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3107{
3108 struct packet_type *pt;
3109 struct list_head *nxt;
3110 int hash;
3111
3112 ++*pos;
3113 if (v == SEQ_START_TOKEN)
3114 return ptype_get_idx(0);
3115
3116 pt = v;
3117 nxt = pt->list.next;
3118 if (pt->type == htons(ETH_P_ALL)) {
3119 if (nxt != &ptype_all)
3120 goto found;
3121 hash = 0;
3122 nxt = ptype_base[0].next;
3123 } else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003124 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003125
3126 while (nxt == &ptype_base[hash]) {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003127 if (++hash >= PTYPE_HASH_SIZE)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003128 return NULL;
3129 nxt = ptype_base[hash].next;
3130 }
3131found:
3132 return list_entry(nxt, struct packet_type, list);
3133}
3134
3135static void ptype_seq_stop(struct seq_file *seq, void *v)
Stephen Hemminger72348a42008-01-21 02:27:29 -08003136 __releases(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003137{
3138 rcu_read_unlock();
3139}
3140
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003141static int ptype_seq_show(struct seq_file *seq, void *v)
3142{
3143 struct packet_type *pt = v;
3144
3145 if (v == SEQ_START_TOKEN)
3146 seq_puts(seq, "Type Device Function\n");
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09003147 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003148 if (pt->type == htons(ETH_P_ALL))
3149 seq_puts(seq, "ALL ");
3150 else
3151 seq_printf(seq, "%04x", ntohs(pt->type));
3152
Alexey Dobriyan908cd2d2008-11-16 19:50:35 -08003153 seq_printf(seq, " %-8s %pF\n",
3154 pt->dev ? pt->dev->name : "", pt->func);
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003155 }
3156
3157 return 0;
3158}
3159
3160static const struct seq_operations ptype_seq_ops = {
3161 .start = ptype_seq_start,
3162 .next = ptype_seq_next,
3163 .stop = ptype_seq_stop,
3164 .show = ptype_seq_show,
3165};
3166
3167static int ptype_seq_open(struct inode *inode, struct file *file)
3168{
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07003169 return seq_open_net(inode, file, &ptype_seq_ops,
3170 sizeof(struct seq_net_private));
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003171}
3172
3173static const struct file_operations ptype_seq_fops = {
3174 .owner = THIS_MODULE,
3175 .open = ptype_seq_open,
3176 .read = seq_read,
3177 .llseek = seq_lseek,
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07003178 .release = seq_release_net,
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003179};
3180
3181
Pavel Emelyanov46650792007-10-08 20:38:39 -07003182static int __net_init dev_proc_net_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003183{
3184 int rc = -ENOMEM;
3185
Eric W. Biederman881d9662007-09-17 11:56:21 -07003186 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003187 goto out;
Eric W. Biederman881d9662007-09-17 11:56:21 -07003188 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003189 goto out_dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07003190 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02003191 goto out_softnet;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003192
Eric W. Biederman881d9662007-09-17 11:56:21 -07003193 if (wext_proc_init(net))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02003194 goto out_ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003195 rc = 0;
3196out:
3197 return rc;
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02003198out_ptype:
Eric W. Biederman881d9662007-09-17 11:56:21 -07003199 proc_net_remove(net, "ptype");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003200out_softnet:
Eric W. Biederman881d9662007-09-17 11:56:21 -07003201 proc_net_remove(net, "softnet_stat");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003202out_dev:
Eric W. Biederman881d9662007-09-17 11:56:21 -07003203 proc_net_remove(net, "dev");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003204 goto out;
3205}
Eric W. Biederman881d9662007-09-17 11:56:21 -07003206
Pavel Emelyanov46650792007-10-08 20:38:39 -07003207static void __net_exit dev_proc_net_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07003208{
3209 wext_proc_exit(net);
3210
3211 proc_net_remove(net, "ptype");
3212 proc_net_remove(net, "softnet_stat");
3213 proc_net_remove(net, "dev");
3214}
3215
Denis V. Lunev022cbae2007-11-13 03:23:50 -08003216static struct pernet_operations __net_initdata dev_proc_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07003217 .init = dev_proc_net_init,
3218 .exit = dev_proc_net_exit,
3219};
3220
3221static int __init dev_proc_init(void)
3222{
3223 return register_pernet_subsys(&dev_proc_ops);
3224}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003225#else
3226#define dev_proc_init() 0
3227#endif /* CONFIG_PROC_FS */
3228
3229
3230/**
3231 * netdev_set_master - set up master/slave pair
3232 * @slave: slave device
3233 * @master: new master device
3234 *
3235 * Changes the master device of the slave. Pass %NULL to break the
3236 * bonding. The caller must hold the RTNL semaphore. On a failure
3237 * a negative errno code is returned. On success the reference counts
3238 * are adjusted, %RTM_NEWLINK is sent to the routing socket and the
3239 * function returns zero.
3240 */
3241int netdev_set_master(struct net_device *slave, struct net_device *master)
3242{
3243 struct net_device *old = slave->master;
3244
3245 ASSERT_RTNL();
3246
3247 if (master) {
3248 if (old)
3249 return -EBUSY;
3250 dev_hold(master);
3251 }
3252
3253 slave->master = master;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003254
Linus Torvalds1da177e2005-04-16 15:20:36 -07003255 synchronize_net();
3256
3257 if (old)
3258 dev_put(old);
3259
3260 if (master)
3261 slave->flags |= IFF_SLAVE;
3262 else
3263 slave->flags &= ~IFF_SLAVE;
3264
3265 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
3266 return 0;
3267}
3268
Patrick McHardyb6c40d62008-10-07 15:26:48 -07003269static void dev_change_rx_flags(struct net_device *dev, int flags)
3270{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003271 const struct net_device_ops *ops = dev->netdev_ops;
3272
3273 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
3274 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07003275}
3276
Wang Chendad9b332008-06-18 01:48:28 -07003277static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07003278{
3279 unsigned short old_flags = dev->flags;
David Howells8192b0c2008-11-14 10:39:10 +11003280 uid_t uid;
3281 gid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07003282
Patrick McHardy24023452007-07-14 18:51:31 -07003283 ASSERT_RTNL();
3284
Wang Chendad9b332008-06-18 01:48:28 -07003285 dev->flags |= IFF_PROMISC;
3286 dev->promiscuity += inc;
3287 if (dev->promiscuity == 0) {
3288 /*
3289 * Avoid overflow.
3290 * If inc causes overflow, untouch promisc and return error.
3291 */
3292 if (inc < 0)
3293 dev->flags &= ~IFF_PROMISC;
3294 else {
3295 dev->promiscuity -= inc;
3296 printk(KERN_WARNING "%s: promiscuity touches roof, "
3297 "set promiscuity failed, promiscuity feature "
3298 "of device might be broken.\n", dev->name);
3299 return -EOVERFLOW;
3300 }
3301 }
Patrick McHardy4417da62007-06-27 01:28:10 -07003302 if (dev->flags != old_flags) {
3303 printk(KERN_INFO "device %s %s promiscuous mode\n",
3304 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
3305 "left");
David Howells8192b0c2008-11-14 10:39:10 +11003306 if (audit_enabled) {
3307 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05003308 audit_log(current->audit_context, GFP_ATOMIC,
3309 AUDIT_ANOM_PROMISCUOUS,
3310 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
3311 dev->name, (dev->flags & IFF_PROMISC),
3312 (old_flags & IFF_PROMISC),
3313 audit_get_loginuid(current),
David Howells8192b0c2008-11-14 10:39:10 +11003314 uid, gid,
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05003315 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11003316 }
Patrick McHardy24023452007-07-14 18:51:31 -07003317
Patrick McHardyb6c40d62008-10-07 15:26:48 -07003318 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07003319 }
Wang Chendad9b332008-06-18 01:48:28 -07003320 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07003321}
3322
Linus Torvalds1da177e2005-04-16 15:20:36 -07003323/**
3324 * dev_set_promiscuity - update promiscuity count on a device
3325 * @dev: device
3326 * @inc: modifier
3327 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07003328 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07003329 * remains above zero the interface remains promiscuous. Once it hits zero
3330 * the device reverts back to normal filtering operation. A negative inc
3331 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07003332 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003333 */
Wang Chendad9b332008-06-18 01:48:28 -07003334int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003335{
3336 unsigned short old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07003337 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003338
Wang Chendad9b332008-06-18 01:48:28 -07003339 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07003340 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07003341 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07003342 if (dev->flags != old_flags)
3343 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07003344 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003345}
3346
3347/**
3348 * dev_set_allmulti - update allmulti count on a device
3349 * @dev: device
3350 * @inc: modifier
3351 *
3352 * Add or remove reception of all multicast frames to a device. While the
3353 * count in the device remains above zero the interface remains listening
3354 * to all interfaces. Once it hits zero the device reverts back to normal
3355 * filtering operation. A negative @inc value is used to drop the counter
3356 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07003357 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003358 */
3359
Wang Chendad9b332008-06-18 01:48:28 -07003360int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003361{
3362 unsigned short old_flags = dev->flags;
3363
Patrick McHardy24023452007-07-14 18:51:31 -07003364 ASSERT_RTNL();
3365
Linus Torvalds1da177e2005-04-16 15:20:36 -07003366 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07003367 dev->allmulti += inc;
3368 if (dev->allmulti == 0) {
3369 /*
3370 * Avoid overflow.
3371 * If inc causes overflow, untouch allmulti and return error.
3372 */
3373 if (inc < 0)
3374 dev->flags &= ~IFF_ALLMULTI;
3375 else {
3376 dev->allmulti -= inc;
3377 printk(KERN_WARNING "%s: allmulti touches roof, "
3378 "set allmulti failed, allmulti feature of "
3379 "device might be broken.\n", dev->name);
3380 return -EOVERFLOW;
3381 }
3382 }
Patrick McHardy24023452007-07-14 18:51:31 -07003383 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07003384 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07003385 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07003386 }
Wang Chendad9b332008-06-18 01:48:28 -07003387 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07003388}
3389
3390/*
3391 * Upload unicast and multicast address lists to device and
3392 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08003393 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07003394 * are present.
3395 */
3396void __dev_set_rx_mode(struct net_device *dev)
3397{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003398 const struct net_device_ops *ops = dev->netdev_ops;
3399
Patrick McHardy4417da62007-06-27 01:28:10 -07003400 /* dev_open will call this function so the list will stay sane. */
3401 if (!(dev->flags&IFF_UP))
3402 return;
3403
3404 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09003405 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07003406
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003407 if (ops->ndo_set_rx_mode)
3408 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07003409 else {
3410 /* Unicast addresses changes may only happen under the rtnl,
3411 * therefore calling __dev_set_promiscuity here is safe.
3412 */
3413 if (dev->uc_count > 0 && !dev->uc_promisc) {
3414 __dev_set_promiscuity(dev, 1);
3415 dev->uc_promisc = 1;
3416 } else if (dev->uc_count == 0 && dev->uc_promisc) {
3417 __dev_set_promiscuity(dev, -1);
3418 dev->uc_promisc = 0;
3419 }
3420
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003421 if (ops->ndo_set_multicast_list)
3422 ops->ndo_set_multicast_list(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07003423 }
3424}
3425
3426void dev_set_rx_mode(struct net_device *dev)
3427{
David S. Millerb9e40852008-07-15 00:15:08 -07003428 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07003429 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07003430 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003431}
3432
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07003433int __dev_addr_delete(struct dev_addr_list **list, int *count,
3434 void *addr, int alen, int glbl)
Patrick McHardybf742482007-06-27 01:26:19 -07003435{
3436 struct dev_addr_list *da;
3437
3438 for (; (da = *list) != NULL; list = &da->next) {
3439 if (memcmp(da->da_addr, addr, da->da_addrlen) == 0 &&
3440 alen == da->da_addrlen) {
3441 if (glbl) {
3442 int old_glbl = da->da_gusers;
3443 da->da_gusers = 0;
3444 if (old_glbl == 0)
3445 break;
3446 }
3447 if (--da->da_users)
3448 return 0;
3449
3450 *list = da->next;
3451 kfree(da);
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07003452 (*count)--;
Patrick McHardybf742482007-06-27 01:26:19 -07003453 return 0;
3454 }
3455 }
3456 return -ENOENT;
3457}
3458
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07003459int __dev_addr_add(struct dev_addr_list **list, int *count,
3460 void *addr, int alen, int glbl)
Patrick McHardybf742482007-06-27 01:26:19 -07003461{
3462 struct dev_addr_list *da;
3463
3464 for (da = *list; da != NULL; da = da->next) {
3465 if (memcmp(da->da_addr, addr, da->da_addrlen) == 0 &&
3466 da->da_addrlen == alen) {
3467 if (glbl) {
3468 int old_glbl = da->da_gusers;
3469 da->da_gusers = 1;
3470 if (old_glbl)
3471 return 0;
3472 }
3473 da->da_users++;
3474 return 0;
3475 }
3476 }
3477
Jorge Boncompte [DTI2]12aa3432008-02-19 14:17:04 -08003478 da = kzalloc(sizeof(*da), GFP_ATOMIC);
Patrick McHardybf742482007-06-27 01:26:19 -07003479 if (da == NULL)
3480 return -ENOMEM;
3481 memcpy(da->da_addr, addr, alen);
3482 da->da_addrlen = alen;
3483 da->da_users = 1;
3484 da->da_gusers = glbl ? 1 : 0;
3485 da->next = *list;
3486 *list = da;
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07003487 (*count)++;
Patrick McHardybf742482007-06-27 01:26:19 -07003488 return 0;
3489}
3490
Patrick McHardy4417da62007-06-27 01:28:10 -07003491/**
3492 * dev_unicast_delete - Release secondary unicast address.
3493 * @dev: device
Randy Dunlap0ed72ec2007-07-26 00:03:29 -07003494 * @addr: address to delete
3495 * @alen: length of @addr
Patrick McHardy4417da62007-06-27 01:28:10 -07003496 *
3497 * Release reference to a secondary unicast address and remove it
Randy Dunlap0ed72ec2007-07-26 00:03:29 -07003498 * from the device if the reference count drops to zero.
Patrick McHardy4417da62007-06-27 01:28:10 -07003499 *
3500 * The caller must hold the rtnl_mutex.
3501 */
3502int dev_unicast_delete(struct net_device *dev, void *addr, int alen)
3503{
3504 int err;
3505
3506 ASSERT_RTNL();
3507
David S. Millerb9e40852008-07-15 00:15:08 -07003508 netif_addr_lock_bh(dev);
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07003509 err = __dev_addr_delete(&dev->uc_list, &dev->uc_count, addr, alen, 0);
3510 if (!err)
Patrick McHardy4417da62007-06-27 01:28:10 -07003511 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07003512 netif_addr_unlock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07003513 return err;
3514}
3515EXPORT_SYMBOL(dev_unicast_delete);
3516
3517/**
3518 * dev_unicast_add - add a secondary unicast address
3519 * @dev: device
Wang Chen5dbaec52008-06-27 19:35:16 -07003520 * @addr: address to add
Randy Dunlap0ed72ec2007-07-26 00:03:29 -07003521 * @alen: length of @addr
Patrick McHardy4417da62007-06-27 01:28:10 -07003522 *
3523 * Add a secondary unicast address to the device or increase
3524 * the reference count if it already exists.
3525 *
3526 * The caller must hold the rtnl_mutex.
3527 */
3528int dev_unicast_add(struct net_device *dev, void *addr, int alen)
3529{
3530 int err;
3531
3532 ASSERT_RTNL();
3533
David S. Millerb9e40852008-07-15 00:15:08 -07003534 netif_addr_lock_bh(dev);
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07003535 err = __dev_addr_add(&dev->uc_list, &dev->uc_count, addr, alen, 0);
3536 if (!err)
Patrick McHardy4417da62007-06-27 01:28:10 -07003537 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07003538 netif_addr_unlock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07003539 return err;
3540}
3541EXPORT_SYMBOL(dev_unicast_add);
3542
Chris Leeche83a2ea2008-01-31 16:53:23 -08003543int __dev_addr_sync(struct dev_addr_list **to, int *to_count,
3544 struct dev_addr_list **from, int *from_count)
3545{
3546 struct dev_addr_list *da, *next;
3547 int err = 0;
3548
3549 da = *from;
3550 while (da != NULL) {
3551 next = da->next;
3552 if (!da->da_synced) {
3553 err = __dev_addr_add(to, to_count,
3554 da->da_addr, da->da_addrlen, 0);
3555 if (err < 0)
3556 break;
3557 da->da_synced = 1;
3558 da->da_users++;
3559 } else if (da->da_users == 1) {
3560 __dev_addr_delete(to, to_count,
3561 da->da_addr, da->da_addrlen, 0);
3562 __dev_addr_delete(from, from_count,
3563 da->da_addr, da->da_addrlen, 0);
3564 }
3565 da = next;
3566 }
3567 return err;
3568}
3569
3570void __dev_addr_unsync(struct dev_addr_list **to, int *to_count,
3571 struct dev_addr_list **from, int *from_count)
3572{
3573 struct dev_addr_list *da, *next;
3574
3575 da = *from;
3576 while (da != NULL) {
3577 next = da->next;
3578 if (da->da_synced) {
3579 __dev_addr_delete(to, to_count,
3580 da->da_addr, da->da_addrlen, 0);
3581 da->da_synced = 0;
3582 __dev_addr_delete(from, from_count,
3583 da->da_addr, da->da_addrlen, 0);
3584 }
3585 da = next;
3586 }
3587}
3588
3589/**
3590 * dev_unicast_sync - Synchronize device's unicast list to another device
3591 * @to: destination device
3592 * @from: source device
3593 *
3594 * Add newly added addresses to the destination device and release
3595 * addresses that have no users left. The source device must be
3596 * locked by netif_tx_lock_bh.
3597 *
3598 * This function is intended to be called from the dev->set_rx_mode
3599 * function of layered software devices.
3600 */
3601int dev_unicast_sync(struct net_device *to, struct net_device *from)
3602{
3603 int err = 0;
3604
David S. Millerb9e40852008-07-15 00:15:08 -07003605 netif_addr_lock_bh(to);
Chris Leeche83a2ea2008-01-31 16:53:23 -08003606 err = __dev_addr_sync(&to->uc_list, &to->uc_count,
3607 &from->uc_list, &from->uc_count);
3608 if (!err)
3609 __dev_set_rx_mode(to);
David S. Millerb9e40852008-07-15 00:15:08 -07003610 netif_addr_unlock_bh(to);
Chris Leeche83a2ea2008-01-31 16:53:23 -08003611 return err;
3612}
3613EXPORT_SYMBOL(dev_unicast_sync);
3614
3615/**
Randy Dunlapbc2cda12008-02-13 15:03:25 -08003616 * dev_unicast_unsync - Remove synchronized addresses from the destination device
Chris Leeche83a2ea2008-01-31 16:53:23 -08003617 * @to: destination device
3618 * @from: source device
3619 *
3620 * Remove all addresses that were added to the destination device by
3621 * dev_unicast_sync(). This function is intended to be called from the
3622 * dev->stop function of layered software devices.
3623 */
3624void dev_unicast_unsync(struct net_device *to, struct net_device *from)
3625{
David S. Millerb9e40852008-07-15 00:15:08 -07003626 netif_addr_lock_bh(from);
David S. Millere308a5d2008-07-15 00:13:44 -07003627 netif_addr_lock(to);
Chris Leeche83a2ea2008-01-31 16:53:23 -08003628
3629 __dev_addr_unsync(&to->uc_list, &to->uc_count,
3630 &from->uc_list, &from->uc_count);
3631 __dev_set_rx_mode(to);
3632
David S. Millere308a5d2008-07-15 00:13:44 -07003633 netif_addr_unlock(to);
David S. Millerb9e40852008-07-15 00:15:08 -07003634 netif_addr_unlock_bh(from);
Chris Leeche83a2ea2008-01-31 16:53:23 -08003635}
3636EXPORT_SYMBOL(dev_unicast_unsync);
3637
Denis Cheng12972622007-07-18 02:12:56 -07003638static void __dev_addr_discard(struct dev_addr_list **list)
3639{
3640 struct dev_addr_list *tmp;
3641
3642 while (*list != NULL) {
3643 tmp = *list;
3644 *list = tmp->next;
3645 if (tmp->da_users > tmp->da_gusers)
3646 printk("__dev_addr_discard: address leakage! "
3647 "da_users=%d\n", tmp->da_users);
3648 kfree(tmp);
3649 }
3650}
3651
Denis Cheng26cc2522007-07-18 02:12:03 -07003652static void dev_addr_discard(struct net_device *dev)
Patrick McHardy4417da62007-06-27 01:28:10 -07003653{
David S. Millerb9e40852008-07-15 00:15:08 -07003654 netif_addr_lock_bh(dev);
Denis Cheng26cc2522007-07-18 02:12:03 -07003655
Patrick McHardy4417da62007-06-27 01:28:10 -07003656 __dev_addr_discard(&dev->uc_list);
3657 dev->uc_count = 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07003658
Denis Cheng456ad752007-07-18 02:10:54 -07003659 __dev_addr_discard(&dev->mc_list);
3660 dev->mc_count = 0;
Denis Cheng26cc2522007-07-18 02:12:03 -07003661
David S. Millerb9e40852008-07-15 00:15:08 -07003662 netif_addr_unlock_bh(dev);
Denis Cheng456ad752007-07-18 02:10:54 -07003663}
3664
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07003665/**
3666 * dev_get_flags - get flags reported to userspace
3667 * @dev: device
3668 *
3669 * Get the combination of flag bits exported through APIs to userspace.
3670 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003671unsigned dev_get_flags(const struct net_device *dev)
3672{
3673 unsigned flags;
3674
3675 flags = (dev->flags & ~(IFF_PROMISC |
3676 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08003677 IFF_RUNNING |
3678 IFF_LOWER_UP |
3679 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07003680 (dev->gflags & (IFF_PROMISC |
3681 IFF_ALLMULTI));
3682
Stefan Rompfb00055a2006-03-20 17:09:11 -08003683 if (netif_running(dev)) {
3684 if (netif_oper_up(dev))
3685 flags |= IFF_RUNNING;
3686 if (netif_carrier_ok(dev))
3687 flags |= IFF_LOWER_UP;
3688 if (netif_dormant(dev))
3689 flags |= IFF_DORMANT;
3690 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003691
3692 return flags;
3693}
3694
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07003695/**
3696 * dev_change_flags - change device settings
3697 * @dev: device
3698 * @flags: device state flags
3699 *
3700 * Change settings on device based state flags. The flags are
3701 * in the userspace exported format.
3702 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003703int dev_change_flags(struct net_device *dev, unsigned flags)
3704{
Thomas Graf7c355f52007-06-05 16:03:03 -07003705 int ret, changes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003706 int old_flags = dev->flags;
3707
Patrick McHardy24023452007-07-14 18:51:31 -07003708 ASSERT_RTNL();
3709
Linus Torvalds1da177e2005-04-16 15:20:36 -07003710 /*
3711 * Set the flags on our device.
3712 */
3713
3714 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
3715 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
3716 IFF_AUTOMEDIA)) |
3717 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
3718 IFF_ALLMULTI));
3719
3720 /*
3721 * Load in the correct multicast list now the flags have changed.
3722 */
3723
Patrick McHardyb6c40d62008-10-07 15:26:48 -07003724 if ((old_flags ^ flags) & IFF_MULTICAST)
3725 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07003726
Patrick McHardy4417da62007-06-27 01:28:10 -07003727 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003728
3729 /*
3730 * Have we downed the interface. We handle IFF_UP ourselves
3731 * according to user attempts to set it, rather than blindly
3732 * setting it.
3733 */
3734
3735 ret = 0;
3736 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
3737 ret = ((old_flags & IFF_UP) ? dev_close : dev_open)(dev);
3738
3739 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07003740 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003741 }
3742
3743 if (dev->flags & IFF_UP &&
3744 ((old_flags ^ dev->flags) &~ (IFF_UP | IFF_PROMISC | IFF_ALLMULTI |
3745 IFF_VOLATILE)))
Pavel Emelyanov056925a2007-09-16 15:42:43 -07003746 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003747
3748 if ((flags ^ dev->gflags) & IFF_PROMISC) {
3749 int inc = (flags & IFF_PROMISC) ? +1 : -1;
3750 dev->gflags ^= IFF_PROMISC;
3751 dev_set_promiscuity(dev, inc);
3752 }
3753
3754 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
3755 is important. Some (broken) drivers set IFF_PROMISC, when
3756 IFF_ALLMULTI is requested not asking us and not reporting.
3757 */
3758 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
3759 int inc = (flags & IFF_ALLMULTI) ? +1 : -1;
3760 dev->gflags ^= IFF_ALLMULTI;
3761 dev_set_allmulti(dev, inc);
3762 }
3763
Thomas Graf7c355f52007-06-05 16:03:03 -07003764 /* Exclude state transition flags, already notified */
3765 changes = (old_flags ^ dev->flags) & ~(IFF_UP | IFF_RUNNING);
3766 if (changes)
3767 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003768
3769 return ret;
3770}
3771
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07003772/**
3773 * dev_set_mtu - Change maximum transfer unit
3774 * @dev: device
3775 * @new_mtu: new transfer unit
3776 *
3777 * Change the maximum transfer size of the network device.
3778 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003779int dev_set_mtu(struct net_device *dev, int new_mtu)
3780{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003781 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003782 int err;
3783
3784 if (new_mtu == dev->mtu)
3785 return 0;
3786
3787 /* MTU must be positive. */
3788 if (new_mtu < 0)
3789 return -EINVAL;
3790
3791 if (!netif_device_present(dev))
3792 return -ENODEV;
3793
3794 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003795 if (ops->ndo_change_mtu)
3796 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003797 else
3798 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003799
Linus Torvalds1da177e2005-04-16 15:20:36 -07003800 if (!err && dev->flags & IFF_UP)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07003801 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003802 return err;
3803}
3804
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07003805/**
3806 * dev_set_mac_address - Change Media Access Control Address
3807 * @dev: device
3808 * @sa: new address
3809 *
3810 * Change the hardware (MAC) address of the device
3811 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003812int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
3813{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003814 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003815 int err;
3816
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003817 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003818 return -EOPNOTSUPP;
3819 if (sa->sa_family != dev->type)
3820 return -EINVAL;
3821 if (!netif_device_present(dev))
3822 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003823 err = ops->ndo_set_mac_address(dev, sa);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003824 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07003825 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003826 return err;
3827}
3828
3829/*
Jeff Garzik14e3e072007-10-08 00:06:32 -07003830 * Perform the SIOCxIFxxx calls, inside read_lock(dev_base_lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003831 */
Jeff Garzik14e3e072007-10-08 00:06:32 -07003832static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003833{
3834 int err;
Eric W. Biederman881d9662007-09-17 11:56:21 -07003835 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003836
3837 if (!dev)
3838 return -ENODEV;
3839
3840 switch (cmd) {
3841 case SIOCGIFFLAGS: /* Get interface flags */
3842 ifr->ifr_flags = dev_get_flags(dev);
3843 return 0;
3844
Linus Torvalds1da177e2005-04-16 15:20:36 -07003845 case SIOCGIFMETRIC: /* Get the metric on the interface
3846 (currently unused) */
3847 ifr->ifr_metric = 0;
3848 return 0;
3849
Linus Torvalds1da177e2005-04-16 15:20:36 -07003850 case SIOCGIFMTU: /* Get the MTU of a device */
3851 ifr->ifr_mtu = dev->mtu;
3852 return 0;
3853
Linus Torvalds1da177e2005-04-16 15:20:36 -07003854 case SIOCGIFHWADDR:
3855 if (!dev->addr_len)
3856 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
3857 else
3858 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
3859 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
3860 ifr->ifr_hwaddr.sa_family = dev->type;
3861 return 0;
3862
Jeff Garzik14e3e072007-10-08 00:06:32 -07003863 case SIOCGIFSLAVE:
3864 err = -EINVAL;
3865 break;
3866
3867 case SIOCGIFMAP:
3868 ifr->ifr_map.mem_start = dev->mem_start;
3869 ifr->ifr_map.mem_end = dev->mem_end;
3870 ifr->ifr_map.base_addr = dev->base_addr;
3871 ifr->ifr_map.irq = dev->irq;
3872 ifr->ifr_map.dma = dev->dma;
3873 ifr->ifr_map.port = dev->if_port;
3874 return 0;
3875
3876 case SIOCGIFINDEX:
3877 ifr->ifr_ifindex = dev->ifindex;
3878 return 0;
3879
3880 case SIOCGIFTXQLEN:
3881 ifr->ifr_qlen = dev->tx_queue_len;
3882 return 0;
3883
3884 default:
3885 /* dev_ioctl() should ensure this case
3886 * is never reached
3887 */
3888 WARN_ON(1);
3889 err = -EINVAL;
3890 break;
3891
3892 }
3893 return err;
3894}
3895
3896/*
3897 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
3898 */
3899static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
3900{
3901 int err;
3902 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08003903 const struct net_device_ops *ops;
Jeff Garzik14e3e072007-10-08 00:06:32 -07003904
3905 if (!dev)
3906 return -ENODEV;
3907
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08003908 ops = dev->netdev_ops;
3909
Jeff Garzik14e3e072007-10-08 00:06:32 -07003910 switch (cmd) {
3911 case SIOCSIFFLAGS: /* Set interface flags */
3912 return dev_change_flags(dev, ifr->ifr_flags);
3913
3914 case SIOCSIFMETRIC: /* Set the metric on the interface
3915 (currently unused) */
3916 return -EOPNOTSUPP;
3917
3918 case SIOCSIFMTU: /* Set the MTU of a device */
3919 return dev_set_mtu(dev, ifr->ifr_mtu);
3920
Linus Torvalds1da177e2005-04-16 15:20:36 -07003921 case SIOCSIFHWADDR:
3922 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
3923
3924 case SIOCSIFHWBROADCAST:
3925 if (ifr->ifr_hwaddr.sa_family != dev->type)
3926 return -EINVAL;
3927 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
3928 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
Pavel Emelyanov056925a2007-09-16 15:42:43 -07003929 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003930 return 0;
3931
Linus Torvalds1da177e2005-04-16 15:20:36 -07003932 case SIOCSIFMAP:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003933 if (ops->ndo_set_config) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003934 if (!netif_device_present(dev))
3935 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003936 return ops->ndo_set_config(dev, &ifr->ifr_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003937 }
3938 return -EOPNOTSUPP;
3939
3940 case SIOCADDMULTI:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003941 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07003942 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
3943 return -EINVAL;
3944 if (!netif_device_present(dev))
3945 return -ENODEV;
3946 return dev_mc_add(dev, ifr->ifr_hwaddr.sa_data,
3947 dev->addr_len, 1);
3948
3949 case SIOCDELMULTI:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003950 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07003951 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
3952 return -EINVAL;
3953 if (!netif_device_present(dev))
3954 return -ENODEV;
3955 return dev_mc_delete(dev, ifr->ifr_hwaddr.sa_data,
3956 dev->addr_len, 1);
3957
Linus Torvalds1da177e2005-04-16 15:20:36 -07003958 case SIOCSIFTXQLEN:
3959 if (ifr->ifr_qlen < 0)
3960 return -EINVAL;
3961 dev->tx_queue_len = ifr->ifr_qlen;
3962 return 0;
3963
3964 case SIOCSIFNAME:
3965 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
3966 return dev_change_name(dev, ifr->ifr_newname);
3967
3968 /*
3969 * Unknown or private ioctl
3970 */
3971
3972 default:
3973 if ((cmd >= SIOCDEVPRIVATE &&
3974 cmd <= SIOCDEVPRIVATE + 15) ||
3975 cmd == SIOCBONDENSLAVE ||
3976 cmd == SIOCBONDRELEASE ||
3977 cmd == SIOCBONDSETHWADDR ||
3978 cmd == SIOCBONDSLAVEINFOQUERY ||
3979 cmd == SIOCBONDINFOQUERY ||
3980 cmd == SIOCBONDCHANGEACTIVE ||
3981 cmd == SIOCGMIIPHY ||
3982 cmd == SIOCGMIIREG ||
3983 cmd == SIOCSMIIREG ||
3984 cmd == SIOCBRADDIF ||
3985 cmd == SIOCBRDELIF ||
Patrick Ohlyd24fff22009-02-12 05:03:40 +00003986 cmd == SIOCSHWTSTAMP ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07003987 cmd == SIOCWANDEV) {
3988 err = -EOPNOTSUPP;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003989 if (ops->ndo_do_ioctl) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003990 if (netif_device_present(dev))
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003991 err = ops->ndo_do_ioctl(dev, ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003992 else
3993 err = -ENODEV;
3994 }
3995 } else
3996 err = -EINVAL;
3997
3998 }
3999 return err;
4000}
4001
4002/*
4003 * This function handles all "interface"-type I/O control requests. The actual
4004 * 'doing' part of this is dev_ifsioc above.
4005 */
4006
4007/**
4008 * dev_ioctl - network device ioctl
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004009 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07004010 * @cmd: command to issue
4011 * @arg: pointer to a struct ifreq in user space
4012 *
4013 * Issue ioctl functions to devices. This is normally called by the
4014 * user space syscall interfaces but can sometimes be useful for
4015 * other purposes. The return value is the return from the syscall if
4016 * positive or a negative errno code on error.
4017 */
4018
Eric W. Biederman881d9662007-09-17 11:56:21 -07004019int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004020{
4021 struct ifreq ifr;
4022 int ret;
4023 char *colon;
4024
4025 /* One special case: SIOCGIFCONF takes ifconf argument
4026 and requires shared lock, because it sleeps writing
4027 to user space.
4028 */
4029
4030 if (cmd == SIOCGIFCONF) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004031 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07004032 ret = dev_ifconf(net, (char __user *) arg);
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004033 rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004034 return ret;
4035 }
4036 if (cmd == SIOCGIFNAME)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004037 return dev_ifname(net, (struct ifreq __user *)arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004038
4039 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4040 return -EFAULT;
4041
4042 ifr.ifr_name[IFNAMSIZ-1] = 0;
4043
4044 colon = strchr(ifr.ifr_name, ':');
4045 if (colon)
4046 *colon = 0;
4047
4048 /*
4049 * See which interface the caller is talking about.
4050 */
4051
4052 switch (cmd) {
4053 /*
4054 * These ioctl calls:
4055 * - can be done by all.
4056 * - atomic and do not require locking.
4057 * - return a value
4058 */
4059 case SIOCGIFFLAGS:
4060 case SIOCGIFMETRIC:
4061 case SIOCGIFMTU:
4062 case SIOCGIFHWADDR:
4063 case SIOCGIFSLAVE:
4064 case SIOCGIFMAP:
4065 case SIOCGIFINDEX:
4066 case SIOCGIFTXQLEN:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004067 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004068 read_lock(&dev_base_lock);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004069 ret = dev_ifsioc_locked(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004070 read_unlock(&dev_base_lock);
4071 if (!ret) {
4072 if (colon)
4073 *colon = ':';
4074 if (copy_to_user(arg, &ifr,
4075 sizeof(struct ifreq)))
4076 ret = -EFAULT;
4077 }
4078 return ret;
4079
4080 case SIOCETHTOOL:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004081 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004082 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07004083 ret = dev_ethtool(net, &ifr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004084 rtnl_unlock();
4085 if (!ret) {
4086 if (colon)
4087 *colon = ':';
4088 if (copy_to_user(arg, &ifr,
4089 sizeof(struct ifreq)))
4090 ret = -EFAULT;
4091 }
4092 return ret;
4093
4094 /*
4095 * These ioctl calls:
4096 * - require superuser power.
4097 * - require strict serialization.
4098 * - return a value
4099 */
4100 case SIOCGMIIPHY:
4101 case SIOCGMIIREG:
4102 case SIOCSIFNAME:
4103 if (!capable(CAP_NET_ADMIN))
4104 return -EPERM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004105 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004106 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07004107 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004108 rtnl_unlock();
4109 if (!ret) {
4110 if (colon)
4111 *colon = ':';
4112 if (copy_to_user(arg, &ifr,
4113 sizeof(struct ifreq)))
4114 ret = -EFAULT;
4115 }
4116 return ret;
4117
4118 /*
4119 * These ioctl calls:
4120 * - require superuser power.
4121 * - require strict serialization.
4122 * - do not return a value
4123 */
4124 case SIOCSIFFLAGS:
4125 case SIOCSIFMETRIC:
4126 case SIOCSIFMTU:
4127 case SIOCSIFMAP:
4128 case SIOCSIFHWADDR:
4129 case SIOCSIFSLAVE:
4130 case SIOCADDMULTI:
4131 case SIOCDELMULTI:
4132 case SIOCSIFHWBROADCAST:
4133 case SIOCSIFTXQLEN:
4134 case SIOCSMIIREG:
4135 case SIOCBONDENSLAVE:
4136 case SIOCBONDRELEASE:
4137 case SIOCBONDSETHWADDR:
Linus Torvalds1da177e2005-04-16 15:20:36 -07004138 case SIOCBONDCHANGEACTIVE:
4139 case SIOCBRADDIF:
4140 case SIOCBRDELIF:
Patrick Ohlyd24fff22009-02-12 05:03:40 +00004141 case SIOCSHWTSTAMP:
Linus Torvalds1da177e2005-04-16 15:20:36 -07004142 if (!capable(CAP_NET_ADMIN))
4143 return -EPERM;
Thomas Grafcabcac02006-01-24 12:46:33 -08004144 /* fall through */
4145 case SIOCBONDSLAVEINFOQUERY:
4146 case SIOCBONDINFOQUERY:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004147 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004148 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07004149 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004150 rtnl_unlock();
4151 return ret;
4152
4153 case SIOCGIFMEM:
4154 /* Get the per device memory space. We can add this but
4155 * currently do not support it */
4156 case SIOCSIFMEM:
4157 /* Set the per device memory buffer space.
4158 * Not applicable in our case */
4159 case SIOCSIFLINK:
4160 return -EINVAL;
4161
4162 /*
4163 * Unknown or private ioctl.
4164 */
4165 default:
4166 if (cmd == SIOCWANDEV ||
4167 (cmd >= SIOCDEVPRIVATE &&
4168 cmd <= SIOCDEVPRIVATE + 15)) {
Eric W. Biederman881d9662007-09-17 11:56:21 -07004169 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004170 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07004171 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004172 rtnl_unlock();
4173 if (!ret && copy_to_user(arg, &ifr,
4174 sizeof(struct ifreq)))
4175 ret = -EFAULT;
4176 return ret;
4177 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004178 /* Take care of Wireless Extensions */
Johannes Berg295f4a12007-04-26 20:43:56 -07004179 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004180 return wext_handle_ioctl(net, &ifr, cmd, arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004181 return -EINVAL;
4182 }
4183}
4184
4185
4186/**
4187 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004188 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07004189 *
4190 * Returns a suitable unique value for a new device interface
4191 * number. The caller must hold the rtnl semaphore or the
4192 * dev_base_lock to be sure it remains unique.
4193 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07004194static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004195{
4196 static int ifindex;
4197 for (;;) {
4198 if (++ifindex <= 0)
4199 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004200 if (!__dev_get_by_index(net, ifindex))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004201 return ifindex;
4202 }
4203}
4204
Linus Torvalds1da177e2005-04-16 15:20:36 -07004205/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08004206static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004207
Stephen Hemminger6f05f622007-03-08 20:46:03 -08004208static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004209{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004210 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004211}
4212
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004213static void rollback_registered(struct net_device *dev)
4214{
4215 BUG_ON(dev_boot_phase);
4216 ASSERT_RTNL();
4217
4218 /* Some devices call without registering for initialization unwind. */
4219 if (dev->reg_state == NETREG_UNINITIALIZED) {
4220 printk(KERN_DEBUG "unregister_netdevice: device %s/%p never "
4221 "was registered\n", dev->name, dev);
4222
4223 WARN_ON(1);
4224 return;
4225 }
4226
4227 BUG_ON(dev->reg_state != NETREG_REGISTERED);
4228
4229 /* If device is running, close it first. */
4230 dev_close(dev);
4231
4232 /* And unlink it from device chain. */
4233 unlist_netdevice(dev);
4234
4235 dev->reg_state = NETREG_UNREGISTERING;
4236
4237 synchronize_net();
4238
4239 /* Shutdown queueing discipline. */
4240 dev_shutdown(dev);
4241
4242
4243 /* Notify protocols, that we are about to destroy
4244 this device. They should clean all the things.
4245 */
4246 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4247
4248 /*
4249 * Flush the unicast and multicast chains
4250 */
4251 dev_addr_discard(dev);
4252
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004253 if (dev->netdev_ops->ndo_uninit)
4254 dev->netdev_ops->ndo_uninit(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004255
4256 /* Notifier chain MUST detach us from master device. */
Ilpo Järvinen547b7922008-07-25 21:43:18 -07004257 WARN_ON(dev->master);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004258
4259 /* Remove entries from kobject tree */
4260 netdev_unregister_kobject(dev);
4261
4262 synchronize_net();
4263
4264 dev_put(dev);
4265}
4266
David S. Millere8a04642008-07-17 00:34:19 -07004267static void __netdev_init_queue_locks_one(struct net_device *dev,
4268 struct netdev_queue *dev_queue,
4269 void *_unused)
David S. Millerc773e842008-07-08 23:13:53 -07004270{
4271 spin_lock_init(&dev_queue->_xmit_lock);
David S. Millercf508b12008-07-22 14:16:42 -07004272 netdev_set_xmit_lockdep_class(&dev_queue->_xmit_lock, dev->type);
David S. Millerc773e842008-07-08 23:13:53 -07004273 dev_queue->xmit_lock_owner = -1;
4274}
4275
4276static void netdev_init_queue_locks(struct net_device *dev)
4277{
David S. Millere8a04642008-07-17 00:34:19 -07004278 netdev_for_each_tx_queue(dev, __netdev_init_queue_locks_one, NULL);
4279 __netdev_init_queue_locks_one(dev, &dev->rx_queue, NULL);
David S. Millerc773e842008-07-08 23:13:53 -07004280}
4281
Herbert Xub63365a2008-10-23 01:11:29 -07004282unsigned long netdev_fix_features(unsigned long features, const char *name)
4283{
4284 /* Fix illegal SG+CSUM combinations. */
4285 if ((features & NETIF_F_SG) &&
4286 !(features & NETIF_F_ALL_CSUM)) {
4287 if (name)
4288 printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "
4289 "checksum feature.\n", name);
4290 features &= ~NETIF_F_SG;
4291 }
4292
4293 /* TSO requires that SG is present as well. */
4294 if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {
4295 if (name)
4296 printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "
4297 "SG feature.\n", name);
4298 features &= ~NETIF_F_TSO;
4299 }
4300
4301 if (features & NETIF_F_UFO) {
4302 if (!(features & NETIF_F_GEN_CSUM)) {
4303 if (name)
4304 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4305 "since no NETIF_F_HW_CSUM feature.\n",
4306 name);
4307 features &= ~NETIF_F_UFO;
4308 }
4309
4310 if (!(features & NETIF_F_SG)) {
4311 if (name)
4312 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4313 "since no NETIF_F_SG feature.\n", name);
4314 features &= ~NETIF_F_UFO;
4315 }
4316 }
4317
4318 return features;
4319}
4320EXPORT_SYMBOL(netdev_fix_features);
4321
David S. Miller9d40bbd2009-03-04 23:46:25 -08004322/* Some devices need to (re-)set their netdev_ops inside
4323 * ->init() or similar. If that happens, we have to setup
4324 * the compat pointers again.
4325 */
4326void netdev_resync_ops(struct net_device *dev)
4327{
4328#ifdef CONFIG_COMPAT_NET_DEV_OPS
4329 const struct net_device_ops *ops = dev->netdev_ops;
4330
4331 dev->init = ops->ndo_init;
4332 dev->uninit = ops->ndo_uninit;
4333 dev->open = ops->ndo_open;
4334 dev->change_rx_flags = ops->ndo_change_rx_flags;
4335 dev->set_rx_mode = ops->ndo_set_rx_mode;
4336 dev->set_multicast_list = ops->ndo_set_multicast_list;
4337 dev->set_mac_address = ops->ndo_set_mac_address;
4338 dev->validate_addr = ops->ndo_validate_addr;
4339 dev->do_ioctl = ops->ndo_do_ioctl;
4340 dev->set_config = ops->ndo_set_config;
4341 dev->change_mtu = ops->ndo_change_mtu;
4342 dev->neigh_setup = ops->ndo_neigh_setup;
4343 dev->tx_timeout = ops->ndo_tx_timeout;
4344 dev->get_stats = ops->ndo_get_stats;
4345 dev->vlan_rx_register = ops->ndo_vlan_rx_register;
4346 dev->vlan_rx_add_vid = ops->ndo_vlan_rx_add_vid;
4347 dev->vlan_rx_kill_vid = ops->ndo_vlan_rx_kill_vid;
4348#ifdef CONFIG_NET_POLL_CONTROLLER
4349 dev->poll_controller = ops->ndo_poll_controller;
4350#endif
4351#endif
4352}
4353EXPORT_SYMBOL(netdev_resync_ops);
4354
Linus Torvalds1da177e2005-04-16 15:20:36 -07004355/**
4356 * register_netdevice - register a network device
4357 * @dev: device to register
4358 *
4359 * Take a completed network device structure and add it to the kernel
4360 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
4361 * chain. 0 is returned on success. A negative errno code is returned
4362 * on a failure to set up the device, or if the name is a duplicate.
4363 *
4364 * Callers must hold the rtnl semaphore. You may want
4365 * register_netdev() instead of this.
4366 *
4367 * BUGS:
4368 * The locking appears insufficient to guarantee two parallel registers
4369 * will not get the same name.
4370 */
4371
4372int register_netdevice(struct net_device *dev)
4373{
4374 struct hlist_head *head;
4375 struct hlist_node *p;
4376 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004377 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004378
4379 BUG_ON(dev_boot_phase);
4380 ASSERT_RTNL();
4381
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004382 might_sleep();
4383
Linus Torvalds1da177e2005-04-16 15:20:36 -07004384 /* When net_device's are persistent, this will be fatal. */
4385 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004386 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004387
David S. Millerf1f28aa2008-07-15 00:08:33 -07004388 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07004389 netdev_set_addr_lockdep_class(dev);
David S. Millerc773e842008-07-08 23:13:53 -07004390 netdev_init_queue_locks(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004391
Linus Torvalds1da177e2005-04-16 15:20:36 -07004392 dev->iflink = -1;
4393
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004394#ifdef CONFIG_COMPAT_NET_DEV_OPS
Marcin Slusarzeb39c572009-04-19 07:24:24 +00004395 /* Netdevice_ops API compatibility support.
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004396 * This is temporary until all network devices are converted.
4397 */
4398 if (dev->netdev_ops) {
David S. Miller9d40bbd2009-03-04 23:46:25 -08004399 netdev_resync_ops(dev);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004400 } else {
4401 char drivername[64];
4402 pr_info("%s (%s): not using net_device_ops yet\n",
4403 dev->name, netdev_drivername(dev, drivername, 64));
4404
4405 /* This works only because net_device_ops and the
Marcin Slusarzeb39c572009-04-19 07:24:24 +00004406 compatibility structure are the same. */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004407 dev->netdev_ops = (void *) &(dev->init);
4408 }
4409#endif
4410
Linus Torvalds1da177e2005-04-16 15:20:36 -07004411 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004412 if (dev->netdev_ops->ndo_init) {
4413 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004414 if (ret) {
4415 if (ret > 0)
4416 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08004417 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004418 }
4419 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004420
Linus Torvalds1da177e2005-04-16 15:20:36 -07004421 if (!dev_valid_name(dev->name)) {
4422 ret = -EINVAL;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07004423 goto err_uninit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004424 }
4425
Eric W. Biederman881d9662007-09-17 11:56:21 -07004426 dev->ifindex = dev_new_index(net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004427 if (dev->iflink == -1)
4428 dev->iflink = dev->ifindex;
4429
4430 /* Check for existence of name */
Eric W. Biederman881d9662007-09-17 11:56:21 -07004431 head = dev_name_hash(net, dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004432 hlist_for_each(p, head) {
4433 struct net_device *d
4434 = hlist_entry(p, struct net_device, name_hlist);
4435 if (!strncmp(d->name, dev->name, IFNAMSIZ)) {
4436 ret = -EEXIST;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07004437 goto err_uninit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004438 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004439 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004440
Stephen Hemmingerd212f872007-06-27 00:47:37 -07004441 /* Fix illegal checksum combinations */
4442 if ((dev->features & NETIF_F_HW_CSUM) &&
4443 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
4444 printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",
4445 dev->name);
4446 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4447 }
4448
4449 if ((dev->features & NETIF_F_NO_CSUM) &&
4450 (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
4451 printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",
4452 dev->name);
4453 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
4454 }
4455
Herbert Xub63365a2008-10-23 01:11:29 -07004456 dev->features = netdev_fix_features(dev->features, dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004457
Lennert Buytenheke5a4a722008-08-03 01:23:10 -07004458 /* Enable software GSO if SG is supported. */
4459 if (dev->features & NETIF_F_SG)
4460 dev->features |= NETIF_F_GSO;
4461
Daniel Lezcanoaaf8cdc2008-05-02 17:00:58 -07004462 netdev_initialize_kobject(dev);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07004463 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004464 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07004465 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004466 dev->reg_state = NETREG_REGISTERED;
4467
Linus Torvalds1da177e2005-04-16 15:20:36 -07004468 /*
4469 * Default initial state at registry is that the
4470 * device is present.
4471 */
4472
4473 set_bit(__LINK_STATE_PRESENT, &dev->state);
4474
Linus Torvalds1da177e2005-04-16 15:20:36 -07004475 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004476 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02004477 list_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004478
4479 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004480 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07004481 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004482 if (ret) {
4483 rollback_registered(dev);
4484 dev->reg_state = NETREG_UNREGISTERED;
4485 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004486
4487out:
4488 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07004489
4490err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004491 if (dev->netdev_ops->ndo_uninit)
4492 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07004493 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004494}
4495
4496/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08004497 * init_dummy_netdev - init a dummy network device for NAPI
4498 * @dev: device to init
4499 *
4500 * This takes a network device structure and initialize the minimum
4501 * amount of fields so it can be used to schedule NAPI polls without
4502 * registering a full blown interface. This is to be used by drivers
4503 * that need to tie several hardware interfaces to a single NAPI
4504 * poll scheduler due to HW limitations.
4505 */
4506int init_dummy_netdev(struct net_device *dev)
4507{
4508 /* Clear everything. Note we don't initialize spinlocks
4509 * are they aren't supposed to be taken by any of the
4510 * NAPI code and this dummy netdev is supposed to be
4511 * only ever used for NAPI polls
4512 */
4513 memset(dev, 0, sizeof(struct net_device));
4514
4515 /* make sure we BUG if trying to hit standard
4516 * register/unregister code path
4517 */
4518 dev->reg_state = NETREG_DUMMY;
4519
4520 /* initialize the ref count */
4521 atomic_set(&dev->refcnt, 1);
4522
4523 /* NAPI wants this */
4524 INIT_LIST_HEAD(&dev->napi_list);
4525
4526 /* a dummy interface is started by default */
4527 set_bit(__LINK_STATE_PRESENT, &dev->state);
4528 set_bit(__LINK_STATE_START, &dev->state);
4529
4530 return 0;
4531}
4532EXPORT_SYMBOL_GPL(init_dummy_netdev);
4533
4534
4535/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07004536 * register_netdev - register a network device
4537 * @dev: device to register
4538 *
4539 * Take a completed network device structure and add it to the kernel
4540 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
4541 * chain. 0 is returned on success. A negative errno code is returned
4542 * on a failure to set up the device, or if the name is a duplicate.
4543 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07004544 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07004545 * and expands the device name if you passed a format string to
4546 * alloc_netdev.
4547 */
4548int register_netdev(struct net_device *dev)
4549{
4550 int err;
4551
4552 rtnl_lock();
4553
4554 /*
4555 * If the name is a format string the caller wants us to do a
4556 * name allocation.
4557 */
4558 if (strchr(dev->name, '%')) {
4559 err = dev_alloc_name(dev, dev->name);
4560 if (err < 0)
4561 goto out;
4562 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004563
Linus Torvalds1da177e2005-04-16 15:20:36 -07004564 err = register_netdevice(dev);
4565out:
4566 rtnl_unlock();
4567 return err;
4568}
4569EXPORT_SYMBOL(register_netdev);
4570
4571/*
4572 * netdev_wait_allrefs - wait until all references are gone.
4573 *
4574 * This is called when unregistering network devices.
4575 *
4576 * Any protocol or device that holds a reference should register
4577 * for netdevice notification, and cleanup and put back the
4578 * reference if they receive an UNREGISTER event.
4579 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004580 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004581 */
4582static void netdev_wait_allrefs(struct net_device *dev)
4583{
4584 unsigned long rebroadcast_time, warning_time;
4585
4586 rebroadcast_time = warning_time = jiffies;
4587 while (atomic_read(&dev->refcnt) != 0) {
4588 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004589 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004590
4591 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004592 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004593
4594 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
4595 &dev->state)) {
4596 /* We must not have linkwatch events
4597 * pending on unregister. If this
4598 * happens, we simply run the queue
4599 * unscheduled, resulting in a noop
4600 * for this device.
4601 */
4602 linkwatch_run_queue();
4603 }
4604
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004605 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004606
4607 rebroadcast_time = jiffies;
4608 }
4609
4610 msleep(250);
4611
4612 if (time_after(jiffies, warning_time + 10 * HZ)) {
4613 printk(KERN_EMERG "unregister_netdevice: "
4614 "waiting for %s to become free. Usage "
4615 "count = %d\n",
4616 dev->name, atomic_read(&dev->refcnt));
4617 warning_time = jiffies;
4618 }
4619 }
4620}
4621
4622/* The sequence is:
4623 *
4624 * rtnl_lock();
4625 * ...
4626 * register_netdevice(x1);
4627 * register_netdevice(x2);
4628 * ...
4629 * unregister_netdevice(y1);
4630 * unregister_netdevice(y2);
4631 * ...
4632 * rtnl_unlock();
4633 * free_netdev(y1);
4634 * free_netdev(y2);
4635 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07004636 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07004637 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004638 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07004639 * without deadlocking with linkwatch via keventd.
4640 * 2) Since we run with the RTNL semaphore not held, we can sleep
4641 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07004642 *
4643 * We must not return until all unregister events added during
4644 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004645 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004646void netdev_run_todo(void)
4647{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07004648 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004649
Linus Torvalds1da177e2005-04-16 15:20:36 -07004650 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07004651 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07004652
4653 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07004654
Linus Torvalds1da177e2005-04-16 15:20:36 -07004655 while (!list_empty(&list)) {
4656 struct net_device *dev
4657 = list_entry(list.next, struct net_device, todo_list);
4658 list_del(&dev->todo_list);
4659
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004660 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004661 printk(KERN_ERR "network todo '%s' but state %d\n",
4662 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004663 dump_stack();
4664 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004665 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004666
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004667 dev->reg_state = NETREG_UNREGISTERED;
4668
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004669 on_each_cpu(flush_backlog, dev, 1);
4670
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004671 netdev_wait_allrefs(dev);
4672
4673 /* paranoia */
4674 BUG_ON(atomic_read(&dev->refcnt));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07004675 WARN_ON(dev->ip_ptr);
4676 WARN_ON(dev->ip6_ptr);
4677 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004678
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07004679 if (dev->destructor)
4680 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07004681
4682 /* Free network device */
4683 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004684 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004685}
4686
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08004687/**
4688 * dev_get_stats - get network device statistics
4689 * @dev: device to get statistics from
4690 *
4691 * Get network statistics from device. The device driver may provide
4692 * its own method by setting dev->netdev_ops->get_stats; otherwise
4693 * the internal statistics structure is used.
4694 */
4695const struct net_device_stats *dev_get_stats(struct net_device *dev)
4696 {
4697 const struct net_device_ops *ops = dev->netdev_ops;
4698
4699 if (ops->ndo_get_stats)
4700 return ops->ndo_get_stats(dev);
4701 else
4702 return &dev->stats;
Rusty Russellc45d2862007-03-28 14:29:08 -07004703}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08004704EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07004705
David S. Millerdc2b4842008-07-08 17:18:23 -07004706static void netdev_init_one_queue(struct net_device *dev,
David S. Millere8a04642008-07-17 00:34:19 -07004707 struct netdev_queue *queue,
4708 void *_unused)
David S. Millerdc2b4842008-07-08 17:18:23 -07004709{
David S. Millerdc2b4842008-07-08 17:18:23 -07004710 queue->dev = dev;
4711}
4712
David S. Millerbb949fb2008-07-08 16:55:56 -07004713static void netdev_init_queues(struct net_device *dev)
4714{
David S. Millere8a04642008-07-17 00:34:19 -07004715 netdev_init_one_queue(dev, &dev->rx_queue, NULL);
4716 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
David S. Millerc3f26a22008-07-31 16:58:50 -07004717 spin_lock_init(&dev->tx_global_lock);
David S. Millerbb949fb2008-07-08 16:55:56 -07004718}
4719
Linus Torvalds1da177e2005-04-16 15:20:36 -07004720/**
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07004721 * alloc_netdev_mq - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004722 * @sizeof_priv: size of private data to allocate space for
4723 * @name: device name format string
4724 * @setup: callback to initialize device
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07004725 * @queue_count: the number of subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07004726 *
4727 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07004728 * and performs basic initialization. Also allocates subquue structs
4729 * for each queue on the device at the end of the netdevice.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004730 */
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07004731struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
4732 void (*setup)(struct net_device *), unsigned int queue_count)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004733{
David S. Millere8a04642008-07-17 00:34:19 -07004734 struct netdev_queue *tx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004735 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07004736 size_t alloc_size;
David S. Millere8a04642008-07-17 00:34:19 -07004737 void *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004738
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07004739 BUG_ON(strlen(name) >= sizeof(dev->name));
4740
David S. Millerfd2ea0a2008-07-17 01:56:23 -07004741 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07004742 if (sizeof_priv) {
4743 /* ensure 32-byte alignment of private area */
4744 alloc_size = (alloc_size + NETDEV_ALIGN_CONST) & ~NETDEV_ALIGN_CONST;
4745 alloc_size += sizeof_priv;
4746 }
4747 /* ensure 32-byte alignment of whole construct */
4748 alloc_size += NETDEV_ALIGN_CONST;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004749
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07004750 p = kzalloc(alloc_size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004751 if (!p) {
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07004752 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004753 return NULL;
4754 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004755
Stephen Hemminger79439862008-07-21 13:28:44 -07004756 tx = kcalloc(queue_count, sizeof(struct netdev_queue), GFP_KERNEL);
David S. Millere8a04642008-07-17 00:34:19 -07004757 if (!tx) {
4758 printk(KERN_ERR "alloc_netdev: Unable to allocate "
4759 "tx qdiscs.\n");
4760 kfree(p);
4761 return NULL;
4762 }
4763
Linus Torvalds1da177e2005-04-16 15:20:36 -07004764 dev = (struct net_device *)
4765 (((long)p + NETDEV_ALIGN_CONST) & ~NETDEV_ALIGN_CONST);
4766 dev->padded = (char *)dev - (char *)p;
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004767 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004768
David S. Millere8a04642008-07-17 00:34:19 -07004769 dev->_tx = tx;
4770 dev->num_tx_queues = queue_count;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07004771 dev->real_num_tx_queues = queue_count;
David S. Millere8a04642008-07-17 00:34:19 -07004772
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07004773 dev->gso_max_size = GSO_MAX_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004774
David S. Millerbb949fb2008-07-08 16:55:56 -07004775 netdev_init_queues(dev);
4776
Herbert Xud565b0a2008-12-15 23:38:52 -08004777 INIT_LIST_HEAD(&dev->napi_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004778 setup(dev);
4779 strcpy(dev->name, name);
4780 return dev;
4781}
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07004782EXPORT_SYMBOL(alloc_netdev_mq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004783
4784/**
4785 * free_netdev - free network device
4786 * @dev: device
4787 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004788 * This function does the last stage of destroying an allocated device
4789 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004790 * If this is the last reference then it will be freed.
4791 */
4792void free_netdev(struct net_device *dev)
4793{
Herbert Xud565b0a2008-12-15 23:38:52 -08004794 struct napi_struct *p, *n;
4795
Denis V. Lunevf3005d72008-04-16 02:02:18 -07004796 release_net(dev_net(dev));
4797
David S. Millere8a04642008-07-17 00:34:19 -07004798 kfree(dev->_tx);
4799
Herbert Xud565b0a2008-12-15 23:38:52 -08004800 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
4801 netif_napi_del(p);
4802
Stephen Hemminger3041a062006-05-26 13:25:24 -07004803 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004804 if (dev->reg_state == NETREG_UNINITIALIZED) {
4805 kfree((char *)dev - dev->padded);
4806 return;
4807 }
4808
4809 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
4810 dev->reg_state = NETREG_RELEASED;
4811
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07004812 /* will free via device release */
4813 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004814}
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004815
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004816/**
4817 * synchronize_net - Synchronize with packet receive processing
4818 *
4819 * Wait for packets currently being received to be done.
4820 * Does not block later packets from starting.
4821 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004822void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004823{
4824 might_sleep();
Paul E. McKenneyfbd568a3e2005-05-01 08:59:04 -07004825 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004826}
4827
4828/**
4829 * unregister_netdevice - remove device from the kernel
4830 * @dev: device
4831 *
4832 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08004833 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004834 *
4835 * Callers must hold the rtnl semaphore. You may want
4836 * unregister_netdev() instead of this.
4837 */
4838
Stephen Hemminger22f8cde2007-02-07 00:09:58 -08004839void unregister_netdevice(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004840{
Herbert Xua6620712007-12-12 19:21:56 -08004841 ASSERT_RTNL();
4842
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004843 rollback_registered(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004844 /* Finish processing unregister after unlock */
4845 net_set_todo(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004846}
4847
4848/**
4849 * unregister_netdev - remove device from the kernel
4850 * @dev: device
4851 *
4852 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08004853 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004854 *
4855 * This is just a wrapper for unregister_netdevice that takes
4856 * the rtnl semaphore. In general you want to use this and not
4857 * unregister_netdevice.
4858 */
4859void unregister_netdev(struct net_device *dev)
4860{
4861 rtnl_lock();
4862 unregister_netdevice(dev);
4863 rtnl_unlock();
4864}
4865
4866EXPORT_SYMBOL(unregister_netdev);
4867
Eric W. Biedermance286d32007-09-12 13:53:49 +02004868/**
4869 * dev_change_net_namespace - move device to different nethost namespace
4870 * @dev: device
4871 * @net: network namespace
4872 * @pat: If not NULL name pattern to try if the current device name
4873 * is already taken in the destination network namespace.
4874 *
4875 * This function shuts down a device interface and moves it
4876 * to a new network namespace. On success 0 is returned, on
4877 * a failure a netagive errno code is returned.
4878 *
4879 * Callers must hold the rtnl semaphore.
4880 */
4881
4882int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
4883{
4884 char buf[IFNAMSIZ];
4885 const char *destname;
4886 int err;
4887
4888 ASSERT_RTNL();
4889
4890 /* Don't allow namespace local devices to be moved. */
4891 err = -EINVAL;
4892 if (dev->features & NETIF_F_NETNS_LOCAL)
4893 goto out;
4894
Eric W. Biederman38918452008-10-27 17:51:47 -07004895#ifdef CONFIG_SYSFS
4896 /* Don't allow real devices to be moved when sysfs
4897 * is enabled.
4898 */
4899 err = -EINVAL;
4900 if (dev->dev.parent)
4901 goto out;
4902#endif
4903
Eric W. Biedermance286d32007-09-12 13:53:49 +02004904 /* Ensure the device has been registrered */
4905 err = -EINVAL;
4906 if (dev->reg_state != NETREG_REGISTERED)
4907 goto out;
4908
4909 /* Get out if there is nothing todo */
4910 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09004911 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02004912 goto out;
4913
4914 /* Pick the destination device name, and ensure
4915 * we can use it in the destination network namespace.
4916 */
4917 err = -EEXIST;
4918 destname = dev->name;
4919 if (__dev_get_by_name(net, destname)) {
4920 /* We get here if we can't use the current device name */
4921 if (!pat)
4922 goto out;
4923 if (!dev_valid_name(pat))
4924 goto out;
4925 if (strchr(pat, '%')) {
4926 if (__dev_alloc_name(net, pat, buf) < 0)
4927 goto out;
4928 destname = buf;
4929 } else
4930 destname = pat;
4931 if (__dev_get_by_name(net, destname))
4932 goto out;
4933 }
4934
4935 /*
4936 * And now a mini version of register_netdevice unregister_netdevice.
4937 */
4938
4939 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07004940 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02004941
4942 /* And unlink it from device chain */
4943 err = -ENODEV;
4944 unlist_netdevice(dev);
4945
4946 synchronize_net();
4947
4948 /* Shutdown queueing discipline. */
4949 dev_shutdown(dev);
4950
4951 /* Notify protocols, that we are about to destroy
4952 this device. They should clean all the things.
4953 */
4954 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4955
4956 /*
4957 * Flush the unicast and multicast chains
4958 */
4959 dev_addr_discard(dev);
4960
Eric W. Biederman38918452008-10-27 17:51:47 -07004961 netdev_unregister_kobject(dev);
4962
Eric W. Biedermance286d32007-09-12 13:53:49 +02004963 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004964 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02004965
4966 /* Assign the new device name */
4967 if (destname != dev->name)
4968 strcpy(dev->name, destname);
4969
4970 /* If there is an ifindex conflict assign a new one */
4971 if (__dev_get_by_index(net, dev->ifindex)) {
4972 int iflink = (dev->iflink == dev->ifindex);
4973 dev->ifindex = dev_new_index(net);
4974 if (iflink)
4975 dev->iflink = dev->ifindex;
4976 }
4977
Eric W. Biederman8b41d182007-09-26 22:02:53 -07004978 /* Fixup kobjects */
Daniel Lezcanoaaf8cdc2008-05-02 17:00:58 -07004979 err = netdev_register_kobject(dev);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07004980 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02004981
4982 /* Add the device back in the hashes */
4983 list_netdevice(dev);
4984
4985 /* Notify protocols, that a new device appeared. */
4986 call_netdevice_notifiers(NETDEV_REGISTER, dev);
4987
4988 synchronize_net();
4989 err = 0;
4990out:
4991 return err;
4992}
4993
Linus Torvalds1da177e2005-04-16 15:20:36 -07004994static int dev_cpu_callback(struct notifier_block *nfb,
4995 unsigned long action,
4996 void *ocpu)
4997{
4998 struct sk_buff **list_skb;
David S. Miller37437bb2008-07-16 02:15:04 -07004999 struct Qdisc **list_net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005000 struct sk_buff *skb;
5001 unsigned int cpu, oldcpu = (unsigned long)ocpu;
5002 struct softnet_data *sd, *oldsd;
5003
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07005004 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005005 return NOTIFY_OK;
5006
5007 local_irq_disable();
5008 cpu = smp_processor_id();
5009 sd = &per_cpu(softnet_data, cpu);
5010 oldsd = &per_cpu(softnet_data, oldcpu);
5011
5012 /* Find end of our completion_queue. */
5013 list_skb = &sd->completion_queue;
5014 while (*list_skb)
5015 list_skb = &(*list_skb)->next;
5016 /* Append completion queue from offline CPU. */
5017 *list_skb = oldsd->completion_queue;
5018 oldsd->completion_queue = NULL;
5019
5020 /* Find end of our output_queue. */
5021 list_net = &sd->output_queue;
5022 while (*list_net)
5023 list_net = &(*list_net)->next_sched;
5024 /* Append output queue from offline CPU. */
5025 *list_net = oldsd->output_queue;
5026 oldsd->output_queue = NULL;
5027
5028 raise_softirq_irqoff(NET_TX_SOFTIRQ);
5029 local_irq_enable();
5030
5031 /* Process offline CPU's input_pkt_queue */
5032 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue)))
5033 netif_rx(skb);
5034
5035 return NOTIFY_OK;
5036}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005037
5038
Herbert Xu7f353bf2007-08-10 15:47:58 -07005039/**
Herbert Xub63365a2008-10-23 01:11:29 -07005040 * netdev_increment_features - increment feature set by one
5041 * @all: current feature set
5042 * @one: new feature set
5043 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07005044 *
5045 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07005046 * @one to the master device with current feature set @all. Will not
5047 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07005048 */
Herbert Xub63365a2008-10-23 01:11:29 -07005049unsigned long netdev_increment_features(unsigned long all, unsigned long one,
5050 unsigned long mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07005051{
Herbert Xub63365a2008-10-23 01:11:29 -07005052 /* If device needs checksumming, downgrade to it. */
5053 if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))
5054 all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);
5055 else if (mask & NETIF_F_ALL_CSUM) {
5056 /* If one device supports v4/v6 checksumming, set for all. */
5057 if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&
5058 !(all & NETIF_F_GEN_CSUM)) {
5059 all &= ~NETIF_F_ALL_CSUM;
5060 all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
5061 }
Herbert Xu7f353bf2007-08-10 15:47:58 -07005062
Herbert Xub63365a2008-10-23 01:11:29 -07005063 /* If one device supports hw checksumming, set for all. */
5064 if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {
5065 all &= ~NETIF_F_ALL_CSUM;
5066 all |= NETIF_F_HW_CSUM;
5067 }
5068 }
Herbert Xu7f353bf2007-08-10 15:47:58 -07005069
Herbert Xub63365a2008-10-23 01:11:29 -07005070 one |= NETIF_F_ALL_CSUM;
Herbert Xu7f353bf2007-08-10 15:47:58 -07005071
Herbert Xub63365a2008-10-23 01:11:29 -07005072 one |= all & NETIF_F_ONE_FOR_ALL;
5073 all &= one | NETIF_F_LLTX | NETIF_F_GSO;
5074 all |= one & mask & NETIF_F_ONE_FOR_ALL;
Herbert Xu7f353bf2007-08-10 15:47:58 -07005075
5076 return all;
5077}
Herbert Xub63365a2008-10-23 01:11:29 -07005078EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07005079
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07005080static struct hlist_head *netdev_create_hash(void)
5081{
5082 int i;
5083 struct hlist_head *hash;
5084
5085 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
5086 if (hash != NULL)
5087 for (i = 0; i < NETDEV_HASHENTRIES; i++)
5088 INIT_HLIST_HEAD(&hash[i]);
5089
5090 return hash;
5091}
5092
Eric W. Biederman881d9662007-09-17 11:56:21 -07005093/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07005094static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07005095{
Eric W. Biederman881d9662007-09-17 11:56:21 -07005096 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07005097
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07005098 net->dev_name_head = netdev_create_hash();
5099 if (net->dev_name_head == NULL)
5100 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005101
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07005102 net->dev_index_head = netdev_create_hash();
5103 if (net->dev_index_head == NULL)
5104 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005105
5106 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07005107
5108err_idx:
5109 kfree(net->dev_name_head);
5110err_name:
5111 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005112}
5113
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005114/**
5115 * netdev_drivername - network driver for the device
5116 * @dev: network device
5117 * @buffer: buffer for resulting name
5118 * @len: size of buffer
5119 *
5120 * Determine network driver for device.
5121 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07005122char *netdev_drivername(const struct net_device *dev, char *buffer, int len)
Arjan van de Ven6579e572008-07-21 13:31:48 -07005123{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07005124 const struct device_driver *driver;
5125 const struct device *parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07005126
5127 if (len <= 0 || !buffer)
5128 return buffer;
5129 buffer[0] = 0;
5130
5131 parent = dev->dev.parent;
5132
5133 if (!parent)
5134 return buffer;
5135
5136 driver = parent->driver;
5137 if (driver && driver->name)
5138 strlcpy(buffer, driver->name, len);
5139 return buffer;
5140}
5141
Pavel Emelyanov46650792007-10-08 20:38:39 -07005142static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07005143{
5144 kfree(net->dev_name_head);
5145 kfree(net->dev_index_head);
5146}
5147
Denis V. Lunev022cbae2007-11-13 03:23:50 -08005148static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07005149 .init = netdev_init,
5150 .exit = netdev_exit,
5151};
5152
Pavel Emelyanov46650792007-10-08 20:38:39 -07005153static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02005154{
Eric W. Biederman8eb79862008-12-29 18:21:48 -08005155 struct net_device *dev;
Eric W. Biedermance286d32007-09-12 13:53:49 +02005156 /*
5157 * Push all migratable of the network devices back to the
5158 * initial network namespace
5159 */
5160 rtnl_lock();
Eric W. Biederman8eb79862008-12-29 18:21:48 -08005161restart:
5162 for_each_netdev(net, dev) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02005163 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07005164 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02005165
5166 /* Ignore unmoveable devices (i.e. loopback) */
5167 if (dev->features & NETIF_F_NETNS_LOCAL)
5168 continue;
5169
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08005170 /* Delete virtual devices */
5171 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) {
5172 dev->rtnl_link_ops->dellink(dev);
Eric W. Biederman8eb79862008-12-29 18:21:48 -08005173 goto restart;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08005174 }
5175
Eric W. Biedermance286d32007-09-12 13:53:49 +02005176 /* Push remaing network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07005177 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
5178 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005179 if (err) {
Pavel Emelyanovaca51392008-05-08 01:24:25 -07005180 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
Eric W. Biedermance286d32007-09-12 13:53:49 +02005181 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07005182 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02005183 }
Eric W. Biederman8eb79862008-12-29 18:21:48 -08005184 goto restart;
Eric W. Biedermance286d32007-09-12 13:53:49 +02005185 }
5186 rtnl_unlock();
5187}
5188
Denis V. Lunev022cbae2007-11-13 03:23:50 -08005189static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02005190 .exit = default_device_exit,
5191};
5192
Linus Torvalds1da177e2005-04-16 15:20:36 -07005193/*
5194 * Initialize the DEV module. At boot time this walks the device list and
5195 * unhooks any devices that fail to initialise (normally hardware not
5196 * present) and leaves us with a valid list of present and active devices.
5197 *
5198 */
5199
5200/*
5201 * This is called single threaded during boot, so no need
5202 * to take the rtnl semaphore.
5203 */
5204static int __init net_dev_init(void)
5205{
5206 int i, rc = -ENOMEM;
5207
5208 BUG_ON(!dev_boot_phase);
5209
Linus Torvalds1da177e2005-04-16 15:20:36 -07005210 if (dev_proc_init())
5211 goto out;
5212
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005213 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07005214 goto out;
5215
5216 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08005217 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005218 INIT_LIST_HEAD(&ptype_base[i]);
5219
Eric W. Biederman881d9662007-09-17 11:56:21 -07005220 if (register_pernet_subsys(&netdev_net_ops))
5221 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005222
5223 /*
5224 * Initialise the packet receive queues.
5225 */
5226
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07005227 for_each_possible_cpu(i) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005228 struct softnet_data *queue;
5229
5230 queue = &per_cpu(softnet_data, i);
5231 skb_queue_head_init(&queue->input_pkt_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005232 queue->completion_queue = NULL;
5233 INIT_LIST_HEAD(&queue->poll_list);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005234
5235 queue->backlog.poll = process_backlog;
5236 queue->backlog.weight = weight_p;
Herbert Xud565b0a2008-12-15 23:38:52 -08005237 queue->backlog.gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00005238 queue->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005239 }
5240
Linus Torvalds1da177e2005-04-16 15:20:36 -07005241 dev_boot_phase = 0;
5242
Eric W. Biederman505d4f72008-11-07 22:54:20 -08005243 /* The loopback device is special if any other network devices
5244 * is present in a network namespace the loopback device must
5245 * be present. Since we now dynamically allocate and free the
5246 * loopback device ensure this invariant is maintained by
5247 * keeping the loopback device as the first device on the
5248 * list of network devices. Ensuring the loopback devices
5249 * is the first device that appears and the last network device
5250 * that disappears.
5251 */
5252 if (register_pernet_device(&loopback_net_ops))
5253 goto out;
5254
5255 if (register_pernet_device(&default_device_ops))
5256 goto out;
5257
Carlos R. Mafra962cf362008-05-15 11:15:37 -03005258 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
5259 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005260
5261 hotcpu_notifier(dev_cpu_callback, 0);
5262 dst_init();
5263 dev_mcast_init();
5264 rc = 0;
5265out:
5266 return rc;
5267}
5268
5269subsys_initcall(net_dev_init);
5270
Krishna Kumare88721f2009-02-18 17:55:02 -08005271static int __init initialize_hashrnd(void)
5272{
5273 get_random_bytes(&skb_tx_hashrnd, sizeof(skb_tx_hashrnd));
5274 return 0;
5275}
5276
5277late_initcall_sync(initialize_hashrnd);
5278
Linus Torvalds1da177e2005-04-16 15:20:36 -07005279EXPORT_SYMBOL(__dev_get_by_index);
5280EXPORT_SYMBOL(__dev_get_by_name);
5281EXPORT_SYMBOL(__dev_remove_pack);
Mitch Williamsc2373ee2005-11-09 10:34:45 -08005282EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005283EXPORT_SYMBOL(dev_add_pack);
5284EXPORT_SYMBOL(dev_alloc_name);
5285EXPORT_SYMBOL(dev_close);
5286EXPORT_SYMBOL(dev_get_by_flags);
5287EXPORT_SYMBOL(dev_get_by_index);
5288EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005289EXPORT_SYMBOL(dev_open);
5290EXPORT_SYMBOL(dev_queue_xmit);
5291EXPORT_SYMBOL(dev_remove_pack);
5292EXPORT_SYMBOL(dev_set_allmulti);
5293EXPORT_SYMBOL(dev_set_promiscuity);
5294EXPORT_SYMBOL(dev_change_flags);
5295EXPORT_SYMBOL(dev_set_mtu);
5296EXPORT_SYMBOL(dev_set_mac_address);
5297EXPORT_SYMBOL(free_netdev);
5298EXPORT_SYMBOL(netdev_boot_setup_check);
5299EXPORT_SYMBOL(netdev_set_master);
5300EXPORT_SYMBOL(netdev_state_change);
5301EXPORT_SYMBOL(netif_receive_skb);
5302EXPORT_SYMBOL(netif_rx);
5303EXPORT_SYMBOL(register_gifconf);
5304EXPORT_SYMBOL(register_netdevice);
5305EXPORT_SYMBOL(register_netdevice_notifier);
5306EXPORT_SYMBOL(skb_checksum_help);
5307EXPORT_SYMBOL(synchronize_net);
5308EXPORT_SYMBOL(unregister_netdevice);
5309EXPORT_SYMBOL(unregister_netdevice_notifier);
5310EXPORT_SYMBOL(net_enable_timestamp);
5311EXPORT_SYMBOL(net_disable_timestamp);
5312EXPORT_SYMBOL(dev_get_flags);
5313
5314#if defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)
5315EXPORT_SYMBOL(br_handle_frame_hook);
5316EXPORT_SYMBOL(br_fdb_get_hook);
5317EXPORT_SYMBOL(br_fdb_put_hook);
5318#endif
5319
Linus Torvalds1da177e2005-04-16 15:20:36 -07005320EXPORT_SYMBOL(dev_load);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005321
5322EXPORT_PER_CPU_SYMBOL(softnet_data);