blob: b35d3bfc66cd66e59466ff743d10d1150194186f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * ROUTE - implementation of the IP router.
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 * Linus Torvalds, <Linus.Torvalds@helsinki.fi>
12 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13 *
14 * Fixes:
15 * Alan Cox : Verify area fixes.
16 * Alan Cox : cli() protects routing changes
17 * Rui Oliveira : ICMP routing table updates
18 * (rco@di.uminho.pt) Routing table insertion and update
19 * Linus Torvalds : Rewrote bits to be sensible
20 * Alan Cox : Added BSD route gw semantics
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +090021 * Alan Cox : Super /proc >4K
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * Alan Cox : MTU in route table
23 * Alan Cox : MSS actually. Also added the window
24 * clamper.
25 * Sam Lantinga : Fixed route matching in rt_del()
26 * Alan Cox : Routing cache support.
27 * Alan Cox : Removed compatibility cruft.
28 * Alan Cox : RTF_REJECT support.
29 * Alan Cox : TCP irtt support.
30 * Jonathan Naylor : Added Metric support.
31 * Miquel van Smoorenburg : BSD API fixes.
32 * Miquel van Smoorenburg : Metrics.
33 * Alan Cox : Use __u32 properly
34 * Alan Cox : Aligned routing errors more closely with BSD
35 * our system is still very different.
36 * Alan Cox : Faster /proc handling
37 * Alexey Kuznetsov : Massive rework to support tree based routing,
38 * routing caches and better behaviour.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +090039 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 * Olaf Erb : irtt wasn't being copied right.
41 * Bjorn Ekwall : Kerneld route support.
42 * Alan Cox : Multicast fixed (I hope)
43 * Pavel Krauz : Limited broadcast fixed
44 * Mike McLagan : Routing by source
45 * Alexey Kuznetsov : End of old history. Split to fib.c and
46 * route.c and rewritten from scratch.
47 * Andi Kleen : Load-limit warning messages.
48 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
49 * Vitaly E. Lavrov : Race condition in ip_route_input_slow.
50 * Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow.
51 * Vladimir V. Ivanov : IP rule info (flowid) is really useful.
52 * Marc Boucher : routing by fwmark
53 * Robert Olsson : Added rt_cache statistics
54 * Arnaldo C. Melo : Convert proc stuff to seq_file
Eric Dumazetbb1d23b2005-07-05 15:00:32 -070055 * Eric Dumazet : hashed spinlocks and rt_check_expire() fixes.
Ilia Sotnikovcef26852006-03-25 01:38:55 -080056 * Ilia Sotnikov : Ignore TOS on PMTUD and Redirect
57 * Ilia Sotnikov : Removed TOS from hash calculations
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 *
59 * This program is free software; you can redistribute it and/or
60 * modify it under the terms of the GNU General Public License
61 * as published by the Free Software Foundation; either version
62 * 2 of the License, or (at your option) any later version.
63 */
64
Joe Perchesafd465032012-03-12 07:03:32 +000065#define pr_fmt(fmt) "IPv4: " fmt
66
Linus Torvalds1da177e2005-04-16 15:20:36 -070067#include <linux/module.h>
68#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070069#include <linux/bitops.h>
70#include <linux/types.h>
71#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070072#include <linux/mm.h>
Eric Dumazet424c4b72005-07-05 14:58:19 -070073#include <linux/bootmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070074#include <linux/string.h>
75#include <linux/socket.h>
76#include <linux/sockios.h>
77#include <linux/errno.h>
78#include <linux/in.h>
79#include <linux/inet.h>
80#include <linux/netdevice.h>
81#include <linux/proc_fs.h>
82#include <linux/init.h>
Eric Dumazet39c90ec2007-09-15 10:55:54 -070083#include <linux/workqueue.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070084#include <linux/skbuff.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/inetdevice.h>
86#include <linux/igmp.h>
87#include <linux/pkt_sched.h>
88#include <linux/mroute.h>
89#include <linux/netfilter_ipv4.h>
90#include <linux/random.h>
91#include <linux/jhash.h>
92#include <linux/rcupdate.h>
93#include <linux/times.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090094#include <linux/slab.h>
Stephen Rothwellb9eda062011-12-22 17:03:29 +110095#include <linux/prefetch.h>
Herbert Xu352e5122007-11-13 21:34:06 -080096#include <net/dst.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/protocol.h>
99#include <net/ip.h>
100#include <net/route.h>
101#include <net/inetpeer.h>
102#include <net/sock.h>
103#include <net/ip_fib.h>
104#include <net/arp.h>
105#include <net/tcp.h>
106#include <net/icmp.h>
107#include <net/xfrm.h>
Tom Tucker8d717402006-07-30 20:43:36 -0700108#include <net/netevent.h>
Thomas Graf63f34442007-03-22 11:55:17 -0700109#include <net/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110#ifdef CONFIG_SYSCTL
111#include <linux/sysctl.h>
Shan Wei7426a562012-04-18 18:05:46 +0000112#include <linux/kmemleak.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113#endif
David S. Miller6e5714e2011-08-03 20:50:44 -0700114#include <net/secure_seq.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115
David S. Miller68a5e3d2011-03-11 20:07:33 -0500116#define RT_FL_TOS(oldflp4) \
Julian Anastasovf61759e2011-12-02 11:39:42 +0000117 ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118
119#define IP_MAX_MTU 0xFFF0
120
121#define RT_GC_TIMEOUT (300*HZ)
122
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123static int ip_rt_max_size;
Stephen Hemminger817bc4d2008-03-22 17:43:59 -0700124static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT;
Eric Dumazet9f28a2f2011-12-21 15:47:16 -0500125static int ip_rt_gc_interval __read_mostly = 60 * HZ;
Stephen Hemminger817bc4d2008-03-22 17:43:59 -0700126static int ip_rt_gc_min_interval __read_mostly = HZ / 2;
127static int ip_rt_redirect_number __read_mostly = 9;
128static int ip_rt_redirect_load __read_mostly = HZ / 50;
129static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
130static int ip_rt_error_cost __read_mostly = HZ;
131static int ip_rt_error_burst __read_mostly = 5 * HZ;
132static int ip_rt_gc_elasticity __read_mostly = 8;
133static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ;
134static int ip_rt_min_pmtu __read_mostly = 512 + 20 + 20;
135static int ip_rt_min_advmss __read_mostly = 256;
Neil Horman1080d702008-10-27 12:28:25 -0700136static int rt_chain_length_max __read_mostly = 20;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137
Eric Dumazet9f28a2f2011-12-21 15:47:16 -0500138static struct delayed_work expires_work;
139static unsigned long expires_ljiffies;
140
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141/*
142 * Interface to generic destination cache.
143 */
144
145static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
David S. Miller0dbaee32010-12-13 12:52:14 -0800146static unsigned int ipv4_default_advmss(const struct dst_entry *dst);
Steffen Klassertebb762f2011-11-23 02:12:51 +0000147static unsigned int ipv4_mtu(const struct dst_entry *dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148static void ipv4_dst_destroy(struct dst_entry *dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
150static void ipv4_link_failure(struct sk_buff *skb);
David S. Miller6700c272012-07-17 03:29:28 -0700151static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
152 struct sk_buff *skb, u32 mtu);
153static void ip_do_redirect(struct dst_entry *dst, struct sock *sk,
154 struct sk_buff *skb);
Daniel Lezcano569d3642008-01-18 03:56:57 -0800155static int rt_garbage_collect(struct dst_ops *ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156
Eric Dumazet72cdd1d2010-11-11 07:14:07 +0000157static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
158 int how)
159{
160}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161
David S. Miller62fa8a82011-01-26 20:51:05 -0800162static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
163{
David S. Miller31248732012-07-10 07:08:18 -0700164 WARN_ON(1);
165 return NULL;
David S. Miller62fa8a82011-01-26 20:51:05 -0800166}
167
David S. Millerf894cbf2012-07-02 21:52:24 -0700168static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
169 struct sk_buff *skb,
170 const void *daddr);
David S. Millerd3aaeb32011-07-18 00:40:17 -0700171
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172static struct dst_ops ipv4_dst_ops = {
173 .family = AF_INET,
Harvey Harrison09640e62009-02-01 00:45:17 -0800174 .protocol = cpu_to_be16(ETH_P_IP),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 .gc = rt_garbage_collect,
176 .check = ipv4_dst_check,
David S. Miller0dbaee32010-12-13 12:52:14 -0800177 .default_advmss = ipv4_default_advmss,
Steffen Klassertebb762f2011-11-23 02:12:51 +0000178 .mtu = ipv4_mtu,
David S. Miller62fa8a82011-01-26 20:51:05 -0800179 .cow_metrics = ipv4_cow_metrics,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180 .destroy = ipv4_dst_destroy,
181 .ifdown = ipv4_dst_ifdown,
182 .negative_advice = ipv4_negative_advice,
183 .link_failure = ipv4_link_failure,
184 .update_pmtu = ip_rt_update_pmtu,
David S. Millere47a1852012-07-11 20:55:47 -0700185 .redirect = ip_do_redirect,
Herbert Xu1ac06e02008-05-20 14:32:14 -0700186 .local_out = __ip_local_out,
David S. Millerd3aaeb32011-07-18 00:40:17 -0700187 .neigh_lookup = ipv4_neigh_lookup,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188};
189
190#define ECN_OR_COST(class) TC_PRIO_##class
191
Philippe De Muyter4839c522007-07-09 15:32:57 -0700192const __u8 ip_tos2prio[16] = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 TC_PRIO_BESTEFFORT,
Dan Siemon4a2b9c32011-03-15 13:56:07 +0000194 ECN_OR_COST(BESTEFFORT),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 TC_PRIO_BESTEFFORT,
196 ECN_OR_COST(BESTEFFORT),
197 TC_PRIO_BULK,
198 ECN_OR_COST(BULK),
199 TC_PRIO_BULK,
200 ECN_OR_COST(BULK),
201 TC_PRIO_INTERACTIVE,
202 ECN_OR_COST(INTERACTIVE),
203 TC_PRIO_INTERACTIVE,
204 ECN_OR_COST(INTERACTIVE),
205 TC_PRIO_INTERACTIVE_BULK,
206 ECN_OR_COST(INTERACTIVE_BULK),
207 TC_PRIO_INTERACTIVE_BULK,
208 ECN_OR_COST(INTERACTIVE_BULK)
209};
Amir Vadaid4a96862012-04-04 21:33:28 +0000210EXPORT_SYMBOL(ip_tos2prio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211
212/*
213 * Route cache.
214 */
215
216/* The locking scheme is rather straight forward:
217 *
218 * 1) Read-Copy Update protects the buckets of the central route hash.
219 * 2) Only writers remove entries, and they hold the lock
220 * as they look at rtable reference counts.
221 * 3) Only readers acquire references to rtable entries,
222 * they do so with atomic increments and with the
223 * lock held.
224 */
225
226struct rt_hash_bucket {
Eric Dumazet1c317202010-10-25 21:02:07 +0000227 struct rtable __rcu *chain;
Eric Dumazet22c047c2005-07-05 14:55:24 -0700228};
Neil Horman1080d702008-10-27 12:28:25 -0700229
Ingo Molnar8a25d5d2006-07-03 00:24:54 -0700230#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
231 defined(CONFIG_PROVE_LOCKING)
Eric Dumazet22c047c2005-07-05 14:55:24 -0700232/*
233 * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
234 * The size of this table is a power of two and depends on the number of CPUS.
Ingo Molnar62051202006-07-03 00:24:59 -0700235 * (on lockdep we have a quite big spinlock_t, so keep the size down there)
Eric Dumazet22c047c2005-07-05 14:55:24 -0700236 */
Ingo Molnar62051202006-07-03 00:24:59 -0700237#ifdef CONFIG_LOCKDEP
238# define RT_HASH_LOCK_SZ 256
Eric Dumazet22c047c2005-07-05 14:55:24 -0700239#else
Ingo Molnar62051202006-07-03 00:24:59 -0700240# if NR_CPUS >= 32
241# define RT_HASH_LOCK_SZ 4096
242# elif NR_CPUS >= 16
243# define RT_HASH_LOCK_SZ 2048
244# elif NR_CPUS >= 8
245# define RT_HASH_LOCK_SZ 1024
246# elif NR_CPUS >= 4
247# define RT_HASH_LOCK_SZ 512
248# else
249# define RT_HASH_LOCK_SZ 256
250# endif
Eric Dumazet22c047c2005-07-05 14:55:24 -0700251#endif
252
253static spinlock_t *rt_hash_locks;
254# define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
Pavel Emelyanov1ff1cc22007-12-05 21:15:05 -0800255
256static __init void rt_hash_lock_init(void)
257{
258 int i;
259
260 rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
261 GFP_KERNEL);
262 if (!rt_hash_locks)
263 panic("IP: failed to allocate rt_hash_locks\n");
264
265 for (i = 0; i < RT_HASH_LOCK_SZ; i++)
266 spin_lock_init(&rt_hash_locks[i]);
267}
Eric Dumazet22c047c2005-07-05 14:55:24 -0700268#else
269# define rt_hash_lock_addr(slot) NULL
Pavel Emelyanov1ff1cc22007-12-05 21:15:05 -0800270
271static inline void rt_hash_lock_init(void)
272{
273}
Eric Dumazet22c047c2005-07-05 14:55:24 -0700274#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275
Stephen Hemminger817bc4d2008-03-22 17:43:59 -0700276static struct rt_hash_bucket *rt_hash_table __read_mostly;
Eric Dumazet95c96172012-04-15 05:58:06 +0000277static unsigned int rt_hash_mask __read_mostly;
Stephen Hemminger817bc4d2008-03-22 17:43:59 -0700278static unsigned int rt_hash_log __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279
Eric Dumazet2f970d82006-01-17 02:54:36 -0800280static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
Eric Dumazet27f39c73e2010-05-19 22:07:23 +0000281#define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282
Denis V. Lunevb00180d2008-07-05 19:04:09 -0700283static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
Eric Dumazet0eae88f2010-04-20 19:06:52 -0700284 int genid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285{
Eric Dumazet0eae88f2010-04-20 19:06:52 -0700286 return jhash_3words((__force u32)daddr, (__force u32)saddr,
Denis V. Lunevb00180d2008-07-05 19:04:09 -0700287 idx, genid)
Eric Dumazet29e75252008-01-31 17:05:09 -0800288 & rt_hash_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289}
290
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700291static inline int rt_genid(struct net *net)
292{
293 return atomic_read(&net->ipv4.rt_genid);
294}
295
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296#ifdef CONFIG_PROC_FS
297struct rt_cache_iter_state {
Denis V. Luneva75e9362008-02-28 20:50:55 -0800298 struct seq_net_private p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 int bucket;
Eric Dumazet29e75252008-01-31 17:05:09 -0800300 int genid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301};
302
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900303static struct rtable *rt_cache_get_first(struct seq_file *seq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304{
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900305 struct rt_cache_iter_state *st = seq->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 struct rtable *r = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307
308 for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
Eric Dumazet33d480c2011-08-11 19:30:52 +0000309 if (!rcu_access_pointer(rt_hash_table[st->bucket].chain))
Eric Dumazeta6272662008-08-28 01:11:25 -0700310 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 rcu_read_lock_bh();
Paul E. McKenneya898def2010-02-22 17:04:49 -0800312 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
Eric Dumazet29e75252008-01-31 17:05:09 -0800313 while (r) {
Changli Gaod8d1f302010-06-10 23:31:35 -0700314 if (dev_net(r->dst.dev) == seq_file_net(seq) &&
Denis V. Luneva75e9362008-02-28 20:50:55 -0800315 r->rt_genid == st->genid)
Eric Dumazet29e75252008-01-31 17:05:09 -0800316 return r;
Changli Gaod8d1f302010-06-10 23:31:35 -0700317 r = rcu_dereference_bh(r->dst.rt_next);
Eric Dumazet29e75252008-01-31 17:05:09 -0800318 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319 rcu_read_unlock_bh();
320 }
Eric Dumazet29e75252008-01-31 17:05:09 -0800321 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322}
323
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900324static struct rtable *__rt_cache_get_next(struct seq_file *seq,
Denis V. Lunev642d6312008-02-28 20:50:33 -0800325 struct rtable *r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326{
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900327 struct rt_cache_iter_state *st = seq->private;
Eric Dumazeta6272662008-08-28 01:11:25 -0700328
Eric Dumazet1c317202010-10-25 21:02:07 +0000329 r = rcu_dereference_bh(r->dst.rt_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330 while (!r) {
331 rcu_read_unlock_bh();
Eric Dumazeta6272662008-08-28 01:11:25 -0700332 do {
333 if (--st->bucket < 0)
334 return NULL;
Eric Dumazet33d480c2011-08-11 19:30:52 +0000335 } while (!rcu_access_pointer(rt_hash_table[st->bucket].chain));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 rcu_read_lock_bh();
Eric Dumazet1c317202010-10-25 21:02:07 +0000337 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 }
Eric Dumazet1c317202010-10-25 21:02:07 +0000339 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340}
341
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900342static struct rtable *rt_cache_get_next(struct seq_file *seq,
Denis V. Lunev642d6312008-02-28 20:50:33 -0800343 struct rtable *r)
344{
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900345 struct rt_cache_iter_state *st = seq->private;
346 while ((r = __rt_cache_get_next(seq, r)) != NULL) {
Changli Gaod8d1f302010-06-10 23:31:35 -0700347 if (dev_net(r->dst.dev) != seq_file_net(seq))
Denis V. Luneva75e9362008-02-28 20:50:55 -0800348 continue;
Denis V. Lunev642d6312008-02-28 20:50:33 -0800349 if (r->rt_genid == st->genid)
350 break;
351 }
352 return r;
353}
354
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900355static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356{
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900357 struct rtable *r = rt_cache_get_first(seq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358
359 if (r)
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900360 while (pos && (r = rt_cache_get_next(seq, r)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 --pos;
362 return pos ? NULL : r;
363}
364
365static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
366{
Eric Dumazet29e75252008-01-31 17:05:09 -0800367 struct rt_cache_iter_state *st = seq->private;
Eric Dumazet29e75252008-01-31 17:05:09 -0800368 if (*pos)
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900369 return rt_cache_get_idx(seq, *pos - 1);
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700370 st->genid = rt_genid(seq_file_net(seq));
Eric Dumazet29e75252008-01-31 17:05:09 -0800371 return SEQ_START_TOKEN;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372}
373
374static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
375{
Eric Dumazet29e75252008-01-31 17:05:09 -0800376 struct rtable *r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377
378 if (v == SEQ_START_TOKEN)
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900379 r = rt_cache_get_first(seq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 else
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900381 r = rt_cache_get_next(seq, v);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 ++*pos;
383 return r;
384}
385
386static void rt_cache_seq_stop(struct seq_file *seq, void *v)
387{
388 if (v && v != SEQ_START_TOKEN)
389 rcu_read_unlock_bh();
390}
391
392static int rt_cache_seq_show(struct seq_file *seq, void *v)
393{
394 if (v == SEQ_START_TOKEN)
395 seq_printf(seq, "%-127s\n",
396 "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
397 "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
398 "HHUptod\tSpecDst");
399 else {
400 struct rtable *r = v;
David S. Miller3c521f22012-07-02 02:04:13 -0700401 int len;
Eric Dumazet218fa902011-11-29 20:05:55 +0000402
Eric Dumazet0eae88f2010-04-20 19:06:52 -0700403 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
David S. Miller794785b2012-07-10 00:52:56 -0700404 "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
405 r->dst.dev ? r->dst.dev->name : "*",
406 (__force u32)r->rt_dst,
407 (__force u32)r->rt_gateway,
408 r->rt_flags, atomic_read(&r->dst.__refcnt),
409 r->dst.__use, 0, (__force u32)r->rt_src,
410 dst_metric_advmss(&r->dst) + 40,
411 dst_metric(&r->dst, RTAX_WINDOW), 0,
412 r->rt_key_tos,
413 -1, 0, 0, &len);
Pavel Emelyanov5e659e42008-04-24 01:02:16 -0700414
415 seq_printf(seq, "%*s\n", 127 - len, "");
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900416 }
417 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418}
419
Stephen Hemmingerf6908082007-03-12 14:34:29 -0700420static const struct seq_operations rt_cache_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 .start = rt_cache_seq_start,
422 .next = rt_cache_seq_next,
423 .stop = rt_cache_seq_stop,
424 .show = rt_cache_seq_show,
425};
426
427static int rt_cache_seq_open(struct inode *inode, struct file *file)
428{
Denis V. Luneva75e9362008-02-28 20:50:55 -0800429 return seq_open_net(inode, file, &rt_cache_seq_ops,
Pavel Emelyanovcf7732e2007-10-10 02:29:29 -0700430 sizeof(struct rt_cache_iter_state));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431}
432
Arjan van de Ven9a321442007-02-12 00:55:35 -0800433static const struct file_operations rt_cache_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 .owner = THIS_MODULE,
435 .open = rt_cache_seq_open,
436 .read = seq_read,
437 .llseek = seq_lseek,
Denis V. Luneva75e9362008-02-28 20:50:55 -0800438 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439};
440
441
442static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
443{
444 int cpu;
445
446 if (*pos == 0)
447 return SEQ_START_TOKEN;
448
Rusty Russell0f23174a2008-12-29 12:23:42 +0000449 for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 if (!cpu_possible(cpu))
451 continue;
452 *pos = cpu+1;
Eric Dumazet2f970d82006-01-17 02:54:36 -0800453 return &per_cpu(rt_cache_stat, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454 }
455 return NULL;
456}
457
458static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
459{
460 int cpu;
461
Rusty Russell0f23174a2008-12-29 12:23:42 +0000462 for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463 if (!cpu_possible(cpu))
464 continue;
465 *pos = cpu+1;
Eric Dumazet2f970d82006-01-17 02:54:36 -0800466 return &per_cpu(rt_cache_stat, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 }
468 return NULL;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900469
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470}
471
472static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
473{
474
475}
476
477static int rt_cpu_seq_show(struct seq_file *seq, void *v)
478{
479 struct rt_cache_stat *st = v;
480
481 if (v == SEQ_START_TOKEN) {
Olaf Rempel5bec0032005-04-28 12:16:08 -0700482 seq_printf(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 return 0;
484 }
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900485
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 seq_printf(seq,"%08x %08x %08x %08x %08x %08x %08x %08x "
487 " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
Eric Dumazetfc66f952010-10-08 06:37:34 +0000488 dst_entries_get_slow(&ipv4_dst_ops),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489 st->in_hit,
490 st->in_slow_tot,
491 st->in_slow_mc,
492 st->in_no_route,
493 st->in_brd,
494 st->in_martian_dst,
495 st->in_martian_src,
496
497 st->out_hit,
498 st->out_slow_tot,
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900499 st->out_slow_mc,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500
501 st->gc_total,
502 st->gc_ignored,
503 st->gc_goal_miss,
504 st->gc_dst_overflow,
505 st->in_hlist_search,
506 st->out_hlist_search
507 );
508 return 0;
509}
510
Stephen Hemmingerf6908082007-03-12 14:34:29 -0700511static const struct seq_operations rt_cpu_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512 .start = rt_cpu_seq_start,
513 .next = rt_cpu_seq_next,
514 .stop = rt_cpu_seq_stop,
515 .show = rt_cpu_seq_show,
516};
517
518
519static int rt_cpu_seq_open(struct inode *inode, struct file *file)
520{
521 return seq_open(file, &rt_cpu_seq_ops);
522}
523
Arjan van de Ven9a321442007-02-12 00:55:35 -0800524static const struct file_operations rt_cpu_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 .owner = THIS_MODULE,
526 .open = rt_cpu_seq_open,
527 .read = seq_read,
528 .llseek = seq_lseek,
529 .release = seq_release,
530};
531
Patrick McHardyc7066f72011-01-14 13:36:42 +0100532#ifdef CONFIG_IP_ROUTE_CLASSID
Alexey Dobriyana661c412009-11-25 15:40:35 -0800533static int rt_acct_proc_show(struct seq_file *m, void *v)
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800534{
Alexey Dobriyana661c412009-11-25 15:40:35 -0800535 struct ip_rt_acct *dst, *src;
536 unsigned int i, j;
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800537
Alexey Dobriyana661c412009-11-25 15:40:35 -0800538 dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
539 if (!dst)
540 return -ENOMEM;
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800541
Alexey Dobriyana661c412009-11-25 15:40:35 -0800542 for_each_possible_cpu(i) {
543 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
544 for (j = 0; j < 256; j++) {
545 dst[j].o_bytes += src[j].o_bytes;
546 dst[j].o_packets += src[j].o_packets;
547 dst[j].i_bytes += src[j].i_bytes;
548 dst[j].i_packets += src[j].i_packets;
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800549 }
550 }
Alexey Dobriyana661c412009-11-25 15:40:35 -0800551
552 seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
553 kfree(dst);
554 return 0;
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800555}
Alexey Dobriyana661c412009-11-25 15:40:35 -0800556
557static int rt_acct_proc_open(struct inode *inode, struct file *file)
558{
559 return single_open(file, rt_acct_proc_show, NULL);
560}
561
562static const struct file_operations rt_acct_proc_fops = {
563 .owner = THIS_MODULE,
564 .open = rt_acct_proc_open,
565 .read = seq_read,
566 .llseek = seq_lseek,
567 .release = single_release,
568};
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800569#endif
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800570
Denis V. Lunev73b38712008-02-28 20:51:18 -0800571static int __net_init ip_rt_do_proc_init(struct net *net)
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800572{
573 struct proc_dir_entry *pde;
574
575 pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
576 &rt_cache_seq_fops);
577 if (!pde)
578 goto err1;
579
Wang Chen77020722008-02-28 14:14:25 -0800580 pde = proc_create("rt_cache", S_IRUGO,
581 net->proc_net_stat, &rt_cpu_seq_fops);
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800582 if (!pde)
583 goto err2;
584
Patrick McHardyc7066f72011-01-14 13:36:42 +0100585#ifdef CONFIG_IP_ROUTE_CLASSID
Alexey Dobriyana661c412009-11-25 15:40:35 -0800586 pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800587 if (!pde)
588 goto err3;
589#endif
590 return 0;
591
Patrick McHardyc7066f72011-01-14 13:36:42 +0100592#ifdef CONFIG_IP_ROUTE_CLASSID
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800593err3:
594 remove_proc_entry("rt_cache", net->proc_net_stat);
595#endif
596err2:
597 remove_proc_entry("rt_cache", net->proc_net);
598err1:
599 return -ENOMEM;
600}
Denis V. Lunev73b38712008-02-28 20:51:18 -0800601
602static void __net_exit ip_rt_do_proc_exit(struct net *net)
603{
604 remove_proc_entry("rt_cache", net->proc_net_stat);
605 remove_proc_entry("rt_cache", net->proc_net);
Patrick McHardyc7066f72011-01-14 13:36:42 +0100606#ifdef CONFIG_IP_ROUTE_CLASSID
Denis V. Lunev73b38712008-02-28 20:51:18 -0800607 remove_proc_entry("rt_acct", net->proc_net);
Alexey Dobriyan0a931ac2010-01-17 03:32:50 +0000608#endif
Denis V. Lunev73b38712008-02-28 20:51:18 -0800609}
610
611static struct pernet_operations ip_rt_proc_ops __net_initdata = {
612 .init = ip_rt_do_proc_init,
613 .exit = ip_rt_do_proc_exit,
614};
615
616static int __init ip_rt_proc_init(void)
617{
618 return register_pernet_subsys(&ip_rt_proc_ops);
619}
620
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800621#else
Denis V. Lunev73b38712008-02-28 20:51:18 -0800622static inline int ip_rt_proc_init(void)
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800623{
624 return 0;
625}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626#endif /* CONFIG_PROC_FS */
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900627
Stephen Hemminger5969f712008-04-10 01:52:09 -0700628static inline void rt_free(struct rtable *rt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629{
Changli Gaod8d1f302010-06-10 23:31:35 -0700630 call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631}
632
Stephen Hemminger5969f712008-04-10 01:52:09 -0700633static inline void rt_drop(struct rtable *rt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 ip_rt_put(rt);
Changli Gaod8d1f302010-06-10 23:31:35 -0700636 call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637}
638
Stephen Hemminger5969f712008-04-10 01:52:09 -0700639static inline int rt_fast_clean(struct rtable *rth)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640{
641 /* Kill broadcast/multicast entries very aggresively, if they
642 collide in hash table with more useful entries */
643 return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
David S. Millerc7537962010-11-11 17:07:48 -0800644 rt_is_input_route(rth) && rth->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645}
646
Stephen Hemminger5969f712008-04-10 01:52:09 -0700647static inline int rt_valuable(struct rtable *rth)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648{
649 return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
David S. Miller59436342012-07-10 06:58:42 -0700650 rth->dst.expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651}
652
653static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
654{
655 unsigned long age;
656 int ret = 0;
657
Changli Gaod8d1f302010-06-10 23:31:35 -0700658 if (atomic_read(&rth->dst.__refcnt))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 goto out;
660
Changli Gaod8d1f302010-06-10 23:31:35 -0700661 age = jiffies - rth->dst.lastuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 if ((age <= tmo1 && !rt_fast_clean(rth)) ||
663 (age <= tmo2 && rt_valuable(rth)))
664 goto out;
665 ret = 1;
666out: return ret;
667}
668
669/* Bits of score are:
670 * 31: very valuable
671 * 30: not quite useless
672 * 29..0: usage counter
673 */
674static inline u32 rt_score(struct rtable *rt)
675{
Changli Gaod8d1f302010-06-10 23:31:35 -0700676 u32 score = jiffies - rt->dst.lastuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677
678 score = ~score & ~(3<<30);
679
680 if (rt_valuable(rt))
681 score |= (1<<31);
682
David S. Millerc7537962010-11-11 17:07:48 -0800683 if (rt_is_output_route(rt) ||
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
685 score |= (1<<30);
686
687 return score;
688}
689
Neil Horman1080d702008-10-27 12:28:25 -0700690static inline bool rt_caching(const struct net *net)
691{
692 return net->ipv4.current_rt_cache_rebuild_count <=
693 net->ipv4.sysctl_rt_cache_rebuild_count;
694}
695
David S. Miller5e2b61f2011-03-04 21:47:09 -0800696static inline bool compare_hash_inputs(const struct rtable *rt1,
697 const struct rtable *rt2)
Neil Horman1080d702008-10-27 12:28:25 -0700698{
David S. Miller5e2b61f2011-03-04 21:47:09 -0800699 return ((((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
700 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
Julian Anastasov97a80412011-08-09 04:01:16 +0000701 (rt1->rt_route_iif ^ rt2->rt_route_iif)) == 0);
Neil Horman1080d702008-10-27 12:28:25 -0700702}
703
David S. Miller5e2b61f2011-03-04 21:47:09 -0800704static inline int compare_keys(struct rtable *rt1, struct rtable *rt2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705{
David S. Miller5e2b61f2011-03-04 21:47:09 -0800706 return (((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
707 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
708 (rt1->rt_mark ^ rt2->rt_mark) |
David S. Miller475949d2011-05-03 19:45:15 -0700709 (rt1->rt_key_tos ^ rt2->rt_key_tos) |
Julian Anastasovd547f722011-08-07 22:20:20 -0700710 (rt1->rt_route_iif ^ rt2->rt_route_iif) |
Julian Anastasov97a80412011-08-09 04:01:16 +0000711 (rt1->rt_oif ^ rt2->rt_oif)) == 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712}
713
Denis V. Lunevb5921912008-01-22 23:50:25 -0800714static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
715{
Changli Gaod8d1f302010-06-10 23:31:35 -0700716 return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
Denis V. Lunevb5921912008-01-22 23:50:25 -0800717}
718
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700719static inline int rt_is_expired(struct rtable *rth)
720{
Changli Gaod8d1f302010-06-10 23:31:35 -0700721 return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700722}
723
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800724/*
725 * Perform a full scan of hash table and free all entries.
726 * Can be called by a softirq or a process.
727 * In the later case, we want to be reschedule if necessary
728 */
David S. Miller6561a3b2010-12-19 21:11:20 -0800729static void rt_do_flush(struct net *net, int process_context)
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800730{
731 unsigned int i;
732 struct rtable *rth, *next;
733
734 for (i = 0; i <= rt_hash_mask; i++) {
David S. Miller6561a3b2010-12-19 21:11:20 -0800735 struct rtable __rcu **pprev;
736 struct rtable *list;
737
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800738 if (process_context && need_resched())
739 cond_resched();
Eric Dumazet33d480c2011-08-11 19:30:52 +0000740 rth = rcu_access_pointer(rt_hash_table[i].chain);
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800741 if (!rth)
742 continue;
743
744 spin_lock_bh(rt_hash_lock_addr(i));
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700745
David S. Miller6561a3b2010-12-19 21:11:20 -0800746 list = NULL;
747 pprev = &rt_hash_table[i].chain;
748 rth = rcu_dereference_protected(*pprev,
Eric Dumazet1c317202010-10-25 21:02:07 +0000749 lockdep_is_held(rt_hash_lock_addr(i)));
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700750
David S. Miller6561a3b2010-12-19 21:11:20 -0800751 while (rth) {
752 next = rcu_dereference_protected(rth->dst.rt_next,
753 lockdep_is_held(rt_hash_lock_addr(i)));
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700754
David S. Miller6561a3b2010-12-19 21:11:20 -0800755 if (!net ||
756 net_eq(dev_net(rth->dst.dev), net)) {
757 rcu_assign_pointer(*pprev, next);
758 rcu_assign_pointer(rth->dst.rt_next, list);
759 list = rth;
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700760 } else {
David S. Miller6561a3b2010-12-19 21:11:20 -0800761 pprev = &rth->dst.rt_next;
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700762 }
David S. Miller6561a3b2010-12-19 21:11:20 -0800763 rth = next;
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700764 }
David S. Miller6561a3b2010-12-19 21:11:20 -0800765
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800766 spin_unlock_bh(rt_hash_lock_addr(i));
767
David S. Miller6561a3b2010-12-19 21:11:20 -0800768 for (; list; list = next) {
769 next = rcu_dereference_protected(list->dst.rt_next, 1);
770 rt_free(list);
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800771 }
772 }
773}
774
Neil Horman1080d702008-10-27 12:28:25 -0700775/*
776 * While freeing expired entries, we compute average chain length
777 * and standard deviation, using fixed-point arithmetic.
778 * This to have an estimation of rt_chain_length_max
779 * rt_chain_length_max = max(elasticity, AVG + 4*SD)
780 * We use 3 bits for frational part, and 29 (or 61) for magnitude.
781 */
782
783#define FRACT_BITS 3
784#define ONE (1UL << FRACT_BITS)
785
Eric Dumazet98376382010-03-08 03:20:00 +0000786/*
787 * Given a hash chain and an item in this hash chain,
788 * find if a previous entry has the same hash_inputs
789 * (but differs on tos, mark or oif)
790 * Returns 0 if an alias is found.
791 * Returns ONE if rth has no alias before itself.
792 */
793static int has_noalias(const struct rtable *head, const struct rtable *rth)
794{
795 const struct rtable *aux = head;
796
797 while (aux != rth) {
David S. Miller5e2b61f2011-03-04 21:47:09 -0800798 if (compare_hash_inputs(aux, rth))
Eric Dumazet98376382010-03-08 03:20:00 +0000799 return 0;
Eric Dumazet1c317202010-10-25 21:02:07 +0000800 aux = rcu_dereference_protected(aux->dst.rt_next, 1);
Eric Dumazet98376382010-03-08 03:20:00 +0000801 }
802 return ONE;
803}
804
Eric Dumazet9f28a2f2011-12-21 15:47:16 -0500805static void rt_check_expire(void)
806{
807 static unsigned int rover;
808 unsigned int i = rover, goal;
809 struct rtable *rth;
810 struct rtable __rcu **rthp;
811 unsigned long samples = 0;
812 unsigned long sum = 0, sum2 = 0;
813 unsigned long delta;
814 u64 mult;
815
816 delta = jiffies - expires_ljiffies;
817 expires_ljiffies = jiffies;
818 mult = ((u64)delta) << rt_hash_log;
819 if (ip_rt_gc_timeout > 1)
820 do_div(mult, ip_rt_gc_timeout);
821 goal = (unsigned int)mult;
822 if (goal > rt_hash_mask)
823 goal = rt_hash_mask + 1;
824 for (; goal > 0; goal--) {
825 unsigned long tmo = ip_rt_gc_timeout;
826 unsigned long length;
827
828 i = (i + 1) & rt_hash_mask;
829 rthp = &rt_hash_table[i].chain;
830
831 if (need_resched())
832 cond_resched();
833
834 samples++;
835
836 if (rcu_dereference_raw(*rthp) == NULL)
837 continue;
838 length = 0;
839 spin_lock_bh(rt_hash_lock_addr(i));
840 while ((rth = rcu_dereference_protected(*rthp,
841 lockdep_is_held(rt_hash_lock_addr(i)))) != NULL) {
842 prefetch(rth->dst.rt_next);
David S. Millerdf67e6c2012-06-26 00:10:09 -0700843 if (rt_is_expired(rth) ||
844 rt_may_expire(rth, tmo, ip_rt_gc_timeout)) {
Eric Dumazet9f28a2f2011-12-21 15:47:16 -0500845 *rthp = rth->dst.rt_next;
846 rt_free(rth);
847 continue;
848 }
Eric Dumazet9f28a2f2011-12-21 15:47:16 -0500849
David S. Millerdf67e6c2012-06-26 00:10:09 -0700850 /* We only count entries on a chain with equal
851 * hash inputs once so that entries for
852 * different QOS levels, and other non-hash
853 * input attributes don't unfairly skew the
854 * length computation
855 */
856 tmo >>= 1;
857 rthp = &rth->dst.rt_next;
858 length += has_noalias(rt_hash_table[i].chain, rth);
Eric Dumazet9f28a2f2011-12-21 15:47:16 -0500859 }
860 spin_unlock_bh(rt_hash_lock_addr(i));
861 sum += length;
862 sum2 += length*length;
863 }
864 if (samples) {
865 unsigned long avg = sum / samples;
866 unsigned long sd = int_sqrt(sum2 / samples - avg*avg);
867 rt_chain_length_max = max_t(unsigned long,
868 ip_rt_gc_elasticity,
869 (avg + 4*sd) >> FRACT_BITS);
870 }
871 rover = i;
872}
873
874/*
875 * rt_worker_func() is run in process context.
876 * we call rt_check_expire() to scan part of the hash table
877 */
878static void rt_worker_func(struct work_struct *work)
879{
880 rt_check_expire();
881 schedule_delayed_work(&expires_work, ip_rt_gc_interval);
882}
883
Eric Dumazet29e75252008-01-31 17:05:09 -0800884/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300885 * Perturbation of rt_genid by a small quantity [1..256]
Eric Dumazet29e75252008-01-31 17:05:09 -0800886 * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
887 * many times (2^24) without giving recent rt_genid.
888 * Jenkins hash is strong enough that litle changes of rt_genid are OK.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 */
Denis V. Lunev86c657f2008-07-05 19:03:31 -0700890static void rt_cache_invalidate(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891{
Eric Dumazet29e75252008-01-31 17:05:09 -0800892 unsigned char shuffle;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893
Eric Dumazet29e75252008-01-31 17:05:09 -0800894 get_random_bytes(&shuffle, sizeof(shuffle));
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700895 atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896}
897
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800898/*
Eric Dumazet29e75252008-01-31 17:05:09 -0800899 * delay < 0 : invalidate cache (fast : entries will be deleted later)
900 * delay >= 0 : invalidate & flush cache (can be long)
901 */
Denis V. Lunev76e6ebf2008-07-05 19:00:44 -0700902void rt_cache_flush(struct net *net, int delay)
Eric Dumazet29e75252008-01-31 17:05:09 -0800903{
Denis V. Lunev86c657f2008-07-05 19:03:31 -0700904 rt_cache_invalidate(net);
Eric Dumazet29e75252008-01-31 17:05:09 -0800905 if (delay >= 0)
David S. Miller6561a3b2010-12-19 21:11:20 -0800906 rt_do_flush(net, !in_softirq());
Eric Dumazet29e75252008-01-31 17:05:09 -0800907}
908
Eric W. Biedermana5ee1552009-11-29 15:45:58 +0000909/* Flush previous cache invalidated entries from the cache */
David S. Miller6561a3b2010-12-19 21:11:20 -0800910void rt_cache_flush_batch(struct net *net)
Eric W. Biedermana5ee1552009-11-29 15:45:58 +0000911{
David S. Miller6561a3b2010-12-19 21:11:20 -0800912 rt_do_flush(net, !in_softirq());
Eric W. Biedermana5ee1552009-11-29 15:45:58 +0000913}
914
Neil Horman1080d702008-10-27 12:28:25 -0700915static void rt_emergency_hash_rebuild(struct net *net)
916{
Joe Perchese87cc472012-05-13 21:56:26 +0000917 net_warn_ratelimited("Route hash chain too long!\n");
Neil Horman3ee94372010-05-08 01:57:52 -0700918 rt_cache_invalidate(net);
Neil Horman1080d702008-10-27 12:28:25 -0700919}
920
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921/*
922 Short description of GC goals.
923
924 We want to build algorithm, which will keep routing cache
925 at some equilibrium point, when number of aged off entries
926 is kept approximately equal to newly generated ones.
927
928 Current expiration strength is variable "expire".
929 We try to adjust it dynamically, so that if networking
930 is idle expires is large enough to keep enough of warm entries,
931 and when load increases it reduces to limit cache size.
932 */
933
Daniel Lezcano569d3642008-01-18 03:56:57 -0800934static int rt_garbage_collect(struct dst_ops *ops)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935{
936 static unsigned long expire = RT_GC_TIMEOUT;
937 static unsigned long last_gc;
938 static int rover;
939 static int equilibrium;
Eric Dumazet1c317202010-10-25 21:02:07 +0000940 struct rtable *rth;
941 struct rtable __rcu **rthp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942 unsigned long now = jiffies;
943 int goal;
Eric Dumazetfc66f952010-10-08 06:37:34 +0000944 int entries = dst_entries_get_fast(&ipv4_dst_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945
946 /*
947 * Garbage collection is pretty expensive,
948 * do not make it too frequently.
949 */
950
951 RT_CACHE_STAT_INC(gc_total);
952
953 if (now - last_gc < ip_rt_gc_min_interval &&
Eric Dumazetfc66f952010-10-08 06:37:34 +0000954 entries < ip_rt_max_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955 RT_CACHE_STAT_INC(gc_ignored);
956 goto out;
957 }
958
Eric Dumazetfc66f952010-10-08 06:37:34 +0000959 entries = dst_entries_get_slow(&ipv4_dst_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960 /* Calculate number of entries, which we want to expire now. */
Eric Dumazetfc66f952010-10-08 06:37:34 +0000961 goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 if (goal <= 0) {
963 if (equilibrium < ipv4_dst_ops.gc_thresh)
964 equilibrium = ipv4_dst_ops.gc_thresh;
Eric Dumazetfc66f952010-10-08 06:37:34 +0000965 goal = entries - equilibrium;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966 if (goal > 0) {
Eric Dumazetb790ced2007-12-21 01:49:07 -0800967 equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
Eric Dumazetfc66f952010-10-08 06:37:34 +0000968 goal = entries - equilibrium;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 }
970 } else {
971 /* We are in dangerous area. Try to reduce cache really
972 * aggressively.
973 */
Eric Dumazetb790ced2007-12-21 01:49:07 -0800974 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
Eric Dumazetfc66f952010-10-08 06:37:34 +0000975 equilibrium = entries - goal;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 }
977
978 if (now - last_gc >= ip_rt_gc_min_interval)
979 last_gc = now;
980
981 if (goal <= 0) {
982 equilibrium += goal;
983 goto work_done;
984 }
985
986 do {
987 int i, k;
988
989 for (i = rt_hash_mask, k = rover; i >= 0; i--) {
990 unsigned long tmo = expire;
991
992 k = (k + 1) & rt_hash_mask;
993 rthp = &rt_hash_table[k].chain;
Eric Dumazet22c047c2005-07-05 14:55:24 -0700994 spin_lock_bh(rt_hash_lock_addr(k));
Eric Dumazet1c317202010-10-25 21:02:07 +0000995 while ((rth = rcu_dereference_protected(*rthp,
996 lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700997 if (!rt_is_expired(rth) &&
Eric Dumazet29e75252008-01-31 17:05:09 -0800998 !rt_may_expire(rth, tmo, expire)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999 tmo >>= 1;
Changli Gaod8d1f302010-06-10 23:31:35 -07001000 rthp = &rth->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 continue;
1002 }
Changli Gaod8d1f302010-06-10 23:31:35 -07001003 *rthp = rth->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004 rt_free(rth);
1005 goal--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006 }
Eric Dumazet22c047c2005-07-05 14:55:24 -07001007 spin_unlock_bh(rt_hash_lock_addr(k));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008 if (goal <= 0)
1009 break;
1010 }
1011 rover = k;
1012
1013 if (goal <= 0)
1014 goto work_done;
1015
1016 /* Goal is not achieved. We stop process if:
1017
1018 - if expire reduced to zero. Otherwise, expire is halfed.
1019 - if table is not full.
1020 - if we are called from interrupt.
1021 - jiffies check is just fallback/debug loop breaker.
1022 We will not spin here for long time in any case.
1023 */
1024
1025 RT_CACHE_STAT_INC(gc_goal_miss);
1026
1027 if (expire == 0)
1028 break;
1029
1030 expire >>= 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031
Eric Dumazetfc66f952010-10-08 06:37:34 +00001032 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033 goto out;
1034 } while (!in_softirq() && time_before_eq(jiffies, now));
1035
Eric Dumazetfc66f952010-10-08 06:37:34 +00001036 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1037 goto out;
1038 if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 goto out;
Joe Perchese87cc472012-05-13 21:56:26 +00001040 net_warn_ratelimited("dst cache overflow\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041 RT_CACHE_STAT_INC(gc_dst_overflow);
1042 return 1;
1043
1044work_done:
1045 expire += ip_rt_gc_min_interval;
1046 if (expire > ip_rt_gc_timeout ||
Eric Dumazetfc66f952010-10-08 06:37:34 +00001047 dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
1048 dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049 expire = ip_rt_gc_timeout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050out: return 0;
1051}
1052
Eric Dumazet98376382010-03-08 03:20:00 +00001053/*
1054 * Returns number of entries in a hash chain that have different hash_inputs
1055 */
1056static int slow_chain_length(const struct rtable *head)
1057{
1058 int length = 0;
1059 const struct rtable *rth = head;
1060
1061 while (rth) {
1062 length += has_noalias(head, rth);
Eric Dumazet1c317202010-10-25 21:02:07 +00001063 rth = rcu_dereference_protected(rth->dst.rt_next, 1);
Eric Dumazet98376382010-03-08 03:20:00 +00001064 }
1065 return length >> FRACT_BITS;
1066}
1067
David S. Millerf894cbf2012-07-02 21:52:24 -07001068static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
1069 struct sk_buff *skb,
1070 const void *daddr)
David Miller3769cff2011-07-11 22:44:24 +00001071{
David S. Millerd3aaeb32011-07-18 00:40:17 -07001072 struct net_device *dev = dst->dev;
1073 const __be32 *pkey = daddr;
David S. Miller39232972012-01-26 15:22:32 -05001074 const struct rtable *rt;
David Miller3769cff2011-07-11 22:44:24 +00001075 struct neighbour *n;
1076
David S. Miller39232972012-01-26 15:22:32 -05001077 rt = (const struct rtable *) dst;
David S. Millera263b302012-07-02 02:02:15 -07001078 if (rt->rt_gateway)
David S. Miller39232972012-01-26 15:22:32 -05001079 pkey = (const __be32 *) &rt->rt_gateway;
David S. Millerf894cbf2012-07-02 21:52:24 -07001080 else if (skb)
1081 pkey = &ip_hdr(skb)->daddr;
David S. Millerd3aaeb32011-07-18 00:40:17 -07001082
David S. Miller80703d22012-02-15 17:48:35 -05001083 n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
David S. Millerd3aaeb32011-07-18 00:40:17 -07001084 if (n)
1085 return n;
David Miller32092ec2011-07-25 00:01:41 +00001086 return neigh_create(&arp_tbl, pkey, dev);
David S. Millerd3aaeb32011-07-18 00:40:17 -07001087}
1088
Eric Dumazet95c96172012-04-15 05:58:06 +00001089static struct rtable *rt_intern_hash(unsigned int hash, struct rtable *rt,
David S. Millerb23dd4f2011-03-02 14:31:35 -08001090 struct sk_buff *skb, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001091{
Eric Dumazet1c317202010-10-25 21:02:07 +00001092 struct rtable *rth, *cand;
1093 struct rtable __rcu **rthp, **candp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 unsigned long now;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095 u32 min_score;
1096 int chain_length;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001097
1098restart:
1099 chain_length = 0;
1100 min_score = ~(u32)0;
1101 cand = NULL;
1102 candp = NULL;
1103 now = jiffies;
1104
Eric Dumazet7586ece2012-06-20 05:02:19 +00001105 if (!rt_caching(dev_net(rt->dst.dev)) || (rt->dst.flags & DST_NOCACHE)) {
Neil Horman73e42892009-06-20 01:15:16 -07001106 /*
1107 * If we're not caching, just tell the caller we
1108 * were successful and don't touch the route. The
1109 * caller hold the sole reference to the cache entry, and
1110 * it will be released when the caller is done with it.
1111 * If we drop it here, the callers have no way to resolve routes
1112 * when we're not caching. Instead, just point *rp at rt, so
1113 * the caller gets a single use out of the route
Neil Hormanb6280b42009-06-22 10:18:53 +00001114 * Note that we do rt_free on this new route entry, so that
1115 * once its refcount hits zero, we are still able to reap it
1116 * (Thanks Alexey)
Eric Dumazet27b75c92010-10-15 05:44:11 +00001117 * Note: To avoid expensive rcu stuff for this uncached dst,
1118 * we set DST_NOCACHE so that dst_release() can free dst without
1119 * waiting a grace period.
Neil Horman73e42892009-06-20 01:15:16 -07001120 */
Neil Hormanb6280b42009-06-22 10:18:53 +00001121
Eric Dumazetc7d44262010-10-03 22:17:54 -07001122 rt->dst.flags |= DST_NOCACHE;
Neil Hormanb6280b42009-06-22 10:18:53 +00001123 goto skip_hashing;
Neil Horman1080d702008-10-27 12:28:25 -07001124 }
1125
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126 rthp = &rt_hash_table[hash].chain;
1127
Eric Dumazet22c047c2005-07-05 14:55:24 -07001128 spin_lock_bh(rt_hash_lock_addr(hash));
Eric Dumazet1c317202010-10-25 21:02:07 +00001129 while ((rth = rcu_dereference_protected(*rthp,
1130 lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
Denis V. Luneve84f84f2008-07-05 19:04:32 -07001131 if (rt_is_expired(rth)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001132 *rthp = rth->dst.rt_next;
Eric Dumazet29e75252008-01-31 17:05:09 -08001133 rt_free(rth);
1134 continue;
1135 }
David S. Miller5e2b61f2011-03-04 21:47:09 -08001136 if (compare_keys(rth, rt) && compare_netns(rth, rt)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 /* Put it first */
Changli Gaod8d1f302010-06-10 23:31:35 -07001138 *rthp = rth->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139 /*
1140 * Since lookup is lockfree, the deletion
1141 * must be visible to another weakly ordered CPU before
1142 * the insertion at the start of the hash chain.
1143 */
Changli Gaod8d1f302010-06-10 23:31:35 -07001144 rcu_assign_pointer(rth->dst.rt_next,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145 rt_hash_table[hash].chain);
1146 /*
1147 * Since lookup is lockfree, the update writes
1148 * must be ordered for consistency on SMP.
1149 */
1150 rcu_assign_pointer(rt_hash_table[hash].chain, rth);
1151
Changli Gaod8d1f302010-06-10 23:31:35 -07001152 dst_use(&rth->dst, now);
Eric Dumazet22c047c2005-07-05 14:55:24 -07001153 spin_unlock_bh(rt_hash_lock_addr(hash));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154
1155 rt_drop(rt);
David S. Millerb23dd4f2011-03-02 14:31:35 -08001156 if (skb)
Changli Gaod8d1f302010-06-10 23:31:35 -07001157 skb_dst_set(skb, &rth->dst);
David S. Millerb23dd4f2011-03-02 14:31:35 -08001158 return rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159 }
1160
Changli Gaod8d1f302010-06-10 23:31:35 -07001161 if (!atomic_read(&rth->dst.__refcnt)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001162 u32 score = rt_score(rth);
1163
1164 if (score <= min_score) {
1165 cand = rth;
1166 candp = rthp;
1167 min_score = score;
1168 }
1169 }
1170
1171 chain_length++;
1172
Changli Gaod8d1f302010-06-10 23:31:35 -07001173 rthp = &rth->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 }
1175
1176 if (cand) {
1177 /* ip_rt_gc_elasticity used to be average length of chain
1178 * length, when exceeded gc becomes really aggressive.
1179 *
1180 * The second limit is less certain. At the moment it allows
1181 * only 2 entries per bucket. We will see.
1182 */
1183 if (chain_length > ip_rt_gc_elasticity) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001184 *candp = cand->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001185 rt_free(cand);
1186 }
Neil Horman1080d702008-10-27 12:28:25 -07001187 } else {
Eric Dumazet98376382010-03-08 03:20:00 +00001188 if (chain_length > rt_chain_length_max &&
1189 slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001190 struct net *net = dev_net(rt->dst.dev);
Neil Horman1080d702008-10-27 12:28:25 -07001191 int num = ++net->ipv4.current_rt_cache_rebuild_count;
Pavel Emelyanovb35ecb52010-03-24 07:43:17 +00001192 if (!rt_caching(net)) {
Joe Perches058bd4d2012-03-11 18:36:11 +00001193 pr_warn("%s: %d rebuilds is over limit, route caching disabled\n",
Changli Gaod8d1f302010-06-10 23:31:35 -07001194 rt->dst.dev->name, num);
Neil Horman1080d702008-10-27 12:28:25 -07001195 }
Pavel Emelyanovb35ecb52010-03-24 07:43:17 +00001196 rt_emergency_hash_rebuild(net);
Pavel Emelyanov6a2bad72010-03-24 21:51:22 +00001197 spin_unlock_bh(rt_hash_lock_addr(hash));
1198
David S. Miller5e2b61f2011-03-04 21:47:09 -08001199 hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
Pavel Emelyanov6a2bad72010-03-24 21:51:22 +00001200 ifindex, rt_genid(net));
1201 goto restart;
Neil Horman1080d702008-10-27 12:28:25 -07001202 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001203 }
1204
Changli Gaod8d1f302010-06-10 23:31:35 -07001205 rt->dst.rt_next = rt_hash_table[hash].chain;
Neil Horman1080d702008-10-27 12:28:25 -07001206
Eric Dumazet00269b52008-10-16 14:18:29 -07001207 /*
1208 * Since lookup is lockfree, we must make sure
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001209 * previous writes to rt are committed to memory
Eric Dumazet00269b52008-10-16 14:18:29 -07001210 * before making rt visible to other CPUS.
1211 */
Eric Dumazet1ddbcb02009-05-19 20:14:28 +00001212 rcu_assign_pointer(rt_hash_table[hash].chain, rt);
Neil Horman1080d702008-10-27 12:28:25 -07001213
Eric Dumazet22c047c2005-07-05 14:55:24 -07001214 spin_unlock_bh(rt_hash_lock_addr(hash));
Neil Horman73e42892009-06-20 01:15:16 -07001215
Neil Hormanb6280b42009-06-22 10:18:53 +00001216skip_hashing:
David S. Millerb23dd4f2011-03-02 14:31:35 -08001217 if (skb)
Changli Gaod8d1f302010-06-10 23:31:35 -07001218 skb_dst_set(skb, &rt->dst);
David S. Millerb23dd4f2011-03-02 14:31:35 -08001219 return rt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220}
1221
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222/*
1223 * Peer allocation may fail only in serious out-of-memory conditions. However
1224 * we still can generate some output.
1225 * Random ID selection looks a bit dangerous because we have no chances to
1226 * select ID being unique in a reasonable period of time.
1227 * But broken packet identifier may be better than no packet at all.
1228 */
1229static void ip_select_fb_ident(struct iphdr *iph)
1230{
1231 static DEFINE_SPINLOCK(ip_fb_id_lock);
1232 static u32 ip_fallback_id;
1233 u32 salt;
1234
1235 spin_lock_bh(&ip_fb_id_lock);
Al Viroe4485152006-09-26 22:15:01 -07001236 salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001237 iph->id = htons(salt & 0xFFFF);
1238 ip_fallback_id = salt;
1239 spin_unlock_bh(&ip_fb_id_lock);
1240}
1241
1242void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
1243{
David S. Miller1d861aa2012-07-10 03:58:16 -07001244 struct net *net = dev_net(dst->dev);
1245 struct inet_peer *peer;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001246
David S. Miller1d861aa2012-07-10 03:58:16 -07001247 peer = inet_getpeer_v4(net->ipv4.peers, iph->daddr, 1);
1248 if (peer) {
1249 iph->id = htons(inet_getid(peer, more));
1250 inet_putpeer(peer);
1251 return;
1252 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253
1254 ip_select_fb_ident(iph);
1255}
Eric Dumazet4bc2f182010-07-09 21:22:10 +00001256EXPORT_SYMBOL(__ip_select_ident);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257
Eric Dumazet95c96172012-04-15 05:58:06 +00001258static void rt_del(unsigned int hash, struct rtable *rt)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259{
Eric Dumazet1c317202010-10-25 21:02:07 +00001260 struct rtable __rcu **rthp;
1261 struct rtable *aux;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262
Eric Dumazet29e75252008-01-31 17:05:09 -08001263 rthp = &rt_hash_table[hash].chain;
Eric Dumazet22c047c2005-07-05 14:55:24 -07001264 spin_lock_bh(rt_hash_lock_addr(hash));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265 ip_rt_put(rt);
Eric Dumazet1c317202010-10-25 21:02:07 +00001266 while ((aux = rcu_dereference_protected(*rthp,
1267 lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
Denis V. Luneve84f84f2008-07-05 19:04:32 -07001268 if (aux == rt || rt_is_expired(aux)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001269 *rthp = aux->dst.rt_next;
Eric Dumazet29e75252008-01-31 17:05:09 -08001270 rt_free(aux);
1271 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272 }
Changli Gaod8d1f302010-06-10 23:31:35 -07001273 rthp = &aux->dst.rt_next;
Eric Dumazet29e75252008-01-31 17:05:09 -08001274 }
Eric Dumazet22c047c2005-07-05 14:55:24 -07001275 spin_unlock_bh(rt_hash_lock_addr(hash));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276}
1277
David S. Miller6700c272012-07-17 03:29:28 -07001278static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001279{
David S. Millere47a1852012-07-11 20:55:47 -07001280 __be32 new_gw = icmp_hdr(skb)->un.gateway;
David S. Miller94206122012-07-11 20:38:08 -07001281 __be32 old_gw = ip_hdr(skb)->saddr;
David S. Millere47a1852012-07-11 20:55:47 -07001282 struct net_device *dev = skb->dev;
David S. Millere47a1852012-07-11 20:55:47 -07001283 struct in_device *in_dev;
1284 struct neighbour *n;
1285 struct rtable *rt;
Denis V. Lunev317805b2008-02-28 20:50:06 -08001286 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287
David S. Miller94206122012-07-11 20:38:08 -07001288 switch (icmp_hdr(skb)->code & 7) {
1289 case ICMP_REDIR_NET:
1290 case ICMP_REDIR_NETTOS:
1291 case ICMP_REDIR_HOST:
1292 case ICMP_REDIR_HOSTTOS:
1293 break;
1294
1295 default:
1296 return;
1297 }
1298
David S. Millere47a1852012-07-11 20:55:47 -07001299 rt = (struct rtable *) dst;
1300 if (rt->rt_gateway != old_gw)
1301 return;
1302
1303 in_dev = __in_dev_get_rcu(dev);
1304 if (!in_dev)
1305 return;
1306
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001307 net = dev_net(dev);
Joe Perches9d4fb272009-11-23 10:41:23 -08001308 if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
1309 ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
1310 ipv4_is_zeronet(new_gw))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311 goto reject_redirect;
1312
1313 if (!IN_DEV_SHARED_MEDIA(in_dev)) {
1314 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
1315 goto reject_redirect;
1316 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
1317 goto reject_redirect;
1318 } else {
Denis V. Lunev317805b2008-02-28 20:50:06 -08001319 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320 goto reject_redirect;
1321 }
1322
David S. Millere47a1852012-07-11 20:55:47 -07001323 n = ipv4_neigh_lookup(dst, NULL, &new_gw);
1324 if (n) {
1325 if (!(n->nud_state & NUD_VALID)) {
1326 neigh_event_send(n, NULL);
1327 } else {
1328 rt->rt_gateway = new_gw;
1329 rt->rt_flags |= RTCF_REDIRECTED;
1330 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
1331 }
1332 neigh_release(n);
1333 }
1334 return;
1335
1336reject_redirect:
1337#ifdef CONFIG_IP_ROUTE_VERBOSE
David S. Miller99ee0382012-07-12 07:40:05 -07001338 if (IN_DEV_LOG_MARTIANS(in_dev)) {
1339 const struct iphdr *iph = (const struct iphdr *) skb->data;
1340 __be32 daddr = iph->daddr;
1341 __be32 saddr = iph->saddr;
1342
David S. Millere47a1852012-07-11 20:55:47 -07001343 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
1344 " Advised path = %pI4 -> %pI4\n",
1345 &old_gw, dev->name, &new_gw,
1346 &saddr, &daddr);
David S. Miller99ee0382012-07-12 07:40:05 -07001347 }
David S. Millere47a1852012-07-11 20:55:47 -07001348#endif
1349 ;
1350}
1351
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
1353{
Eric Dumazetee6b9672008-03-05 18:30:47 -08001354 struct rtable *rt = (struct rtable *)dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355 struct dst_entry *ret = dst;
1356
1357 if (rt) {
Timo Teräsd11a4dc2010-03-18 23:20:20 +00001358 if (dst->obsolete > 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359 ip_rt_put(rt);
1360 ret = NULL;
David S. Miller59436342012-07-10 06:58:42 -07001361 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
1362 rt->dst.expires) {
Eric Dumazet95c96172012-04-15 05:58:06 +00001363 unsigned int hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
David S. Miller5e2b61f2011-03-04 21:47:09 -08001364 rt->rt_oif,
Denis V. Luneve84f84f2008-07-05 19:04:32 -07001365 rt_genid(dev_net(dst->dev)));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366 rt_del(hash, rt);
1367 ret = NULL;
1368 }
1369 }
1370 return ret;
1371}
1372
1373/*
1374 * Algorithm:
1375 * 1. The first ip_rt_redirect_number redirects are sent
1376 * with exponential backoff, then we stop sending them at all,
1377 * assuming that the host ignores our redirects.
1378 * 2. If we did not see packets requiring redirects
1379 * during ip_rt_redirect_silence, we assume that the host
1380 * forgot redirected route and start to send redirects again.
1381 *
1382 * This algorithm is much cheaper and more intelligent than dumb load limiting
1383 * in icmp.c.
1384 *
1385 * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
1386 * and "frag. need" (breaks PMTU discovery) in icmp.c.
1387 */
1388
1389void ip_rt_send_redirect(struct sk_buff *skb)
1390{
Eric Dumazet511c3f92009-06-02 05:14:27 +00001391 struct rtable *rt = skb_rtable(skb);
Eric Dumazet30038fc2009-08-28 23:52:01 -07001392 struct in_device *in_dev;
David S. Miller92d86822011-02-04 15:55:25 -08001393 struct inet_peer *peer;
David S. Miller1d861aa2012-07-10 03:58:16 -07001394 struct net *net;
Eric Dumazet30038fc2009-08-28 23:52:01 -07001395 int log_martians;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396
Eric Dumazet30038fc2009-08-28 23:52:01 -07001397 rcu_read_lock();
Changli Gaod8d1f302010-06-10 23:31:35 -07001398 in_dev = __in_dev_get_rcu(rt->dst.dev);
Eric Dumazet30038fc2009-08-28 23:52:01 -07001399 if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
1400 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401 return;
Eric Dumazet30038fc2009-08-28 23:52:01 -07001402 }
1403 log_martians = IN_DEV_LOG_MARTIANS(in_dev);
1404 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405
David S. Miller1d861aa2012-07-10 03:58:16 -07001406 net = dev_net(rt->dst.dev);
1407 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
David S. Miller92d86822011-02-04 15:55:25 -08001408 if (!peer) {
1409 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1410 return;
1411 }
1412
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413 /* No redirected packets during ip_rt_redirect_silence;
1414 * reset the algorithm.
1415 */
David S. Miller92d86822011-02-04 15:55:25 -08001416 if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
1417 peer->rate_tokens = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418
1419 /* Too many ignored redirects; do not send anything
Changli Gaod8d1f302010-06-10 23:31:35 -07001420 * set dst.rate_last to the last seen redirected packet.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421 */
David S. Miller92d86822011-02-04 15:55:25 -08001422 if (peer->rate_tokens >= ip_rt_redirect_number) {
1423 peer->rate_last = jiffies;
David S. Miller1d861aa2012-07-10 03:58:16 -07001424 goto out_put_peer;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425 }
1426
1427 /* Check for load limit; set rate_last to the latest sent
1428 * redirect.
1429 */
David S. Miller92d86822011-02-04 15:55:25 -08001430 if (peer->rate_tokens == 0 ||
Li Yewang14fb8a72006-12-18 00:26:35 -08001431 time_after(jiffies,
David S. Miller92d86822011-02-04 15:55:25 -08001432 (peer->rate_last +
1433 (ip_rt_redirect_load << peer->rate_tokens)))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001434 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
David S. Miller92d86822011-02-04 15:55:25 -08001435 peer->rate_last = jiffies;
1436 ++peer->rate_tokens;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001437#ifdef CONFIG_IP_ROUTE_VERBOSE
Eric Dumazet30038fc2009-08-28 23:52:01 -07001438 if (log_martians &&
Joe Perchese87cc472012-05-13 21:56:26 +00001439 peer->rate_tokens == ip_rt_redirect_number)
1440 net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
1441 &ip_hdr(skb)->saddr, rt->rt_iif,
1442 &rt->rt_dst, &rt->rt_gateway);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443#endif
1444 }
David S. Miller1d861aa2012-07-10 03:58:16 -07001445out_put_peer:
1446 inet_putpeer(peer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447}
1448
1449static int ip_error(struct sk_buff *skb)
1450{
David S. Miller251da412012-06-26 16:27:09 -07001451 struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
Eric Dumazet511c3f92009-06-02 05:14:27 +00001452 struct rtable *rt = skb_rtable(skb);
David S. Miller92d86822011-02-04 15:55:25 -08001453 struct inet_peer *peer;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454 unsigned long now;
David S. Miller251da412012-06-26 16:27:09 -07001455 struct net *net;
David S. Miller92d86822011-02-04 15:55:25 -08001456 bool send;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457 int code;
1458
David S. Miller251da412012-06-26 16:27:09 -07001459 net = dev_net(rt->dst.dev);
1460 if (!IN_DEV_FORWARD(in_dev)) {
1461 switch (rt->dst.error) {
1462 case EHOSTUNREACH:
1463 IP_INC_STATS_BH(net, IPSTATS_MIB_INADDRERRORS);
1464 break;
1465
1466 case ENETUNREACH:
1467 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
1468 break;
1469 }
1470 goto out;
1471 }
1472
Changli Gaod8d1f302010-06-10 23:31:35 -07001473 switch (rt->dst.error) {
Joe Perches4500ebf2011-07-01 09:43:07 +00001474 case EINVAL:
1475 default:
1476 goto out;
1477 case EHOSTUNREACH:
1478 code = ICMP_HOST_UNREACH;
1479 break;
1480 case ENETUNREACH:
1481 code = ICMP_NET_UNREACH;
David S. Miller251da412012-06-26 16:27:09 -07001482 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
Joe Perches4500ebf2011-07-01 09:43:07 +00001483 break;
1484 case EACCES:
1485 code = ICMP_PKT_FILTERED;
1486 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487 }
1488
David S. Miller1d861aa2012-07-10 03:58:16 -07001489 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
David S. Miller92d86822011-02-04 15:55:25 -08001490
1491 send = true;
1492 if (peer) {
1493 now = jiffies;
1494 peer->rate_tokens += now - peer->rate_last;
1495 if (peer->rate_tokens > ip_rt_error_burst)
1496 peer->rate_tokens = ip_rt_error_burst;
1497 peer->rate_last = now;
1498 if (peer->rate_tokens >= ip_rt_error_cost)
1499 peer->rate_tokens -= ip_rt_error_cost;
1500 else
1501 send = false;
David S. Miller1d861aa2012-07-10 03:58:16 -07001502 inet_putpeer(peer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503 }
David S. Miller92d86822011-02-04 15:55:25 -08001504 if (send)
1505 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506
1507out: kfree_skb(skb);
1508 return 0;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001509}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510
David S. Miller6700c272012-07-17 03:29:28 -07001511static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
1512 struct sk_buff *skb, u32 mtu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001513{
David S. Miller2c8cec52011-02-09 20:42:07 -08001514 struct rtable *rt = (struct rtable *) dst;
David S. Miller2c8cec52011-02-09 20:42:07 -08001515
1516 dst_confirm(dst);
1517
David S. Miller59436342012-07-10 06:58:42 -07001518 if (mtu < ip_rt_min_pmtu)
1519 mtu = ip_rt_min_pmtu;
Eric Dumazetfe6fe792011-06-08 06:07:07 +00001520
David S. Miller59436342012-07-10 06:58:42 -07001521 rt->rt_pmtu = mtu;
1522 dst_set_expires(&rt->dst, ip_rt_mtu_expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523}
1524
David S. Miller36393392012-06-14 22:21:46 -07001525void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1526 int oif, u32 mark, u8 protocol, int flow_flags)
1527{
1528 const struct iphdr *iph = (const struct iphdr *)skb->data;
1529 struct flowi4 fl4;
1530 struct rtable *rt;
1531
1532 flowi4_init_output(&fl4, oif, mark, RT_TOS(iph->tos), RT_SCOPE_UNIVERSE,
David S. Miller3e129392012-07-10 04:01:57 -07001533 protocol, flow_flags,
David S. Miller36393392012-06-14 22:21:46 -07001534 iph->daddr, iph->saddr, 0, 0);
1535 rt = __ip_route_output_key(net, &fl4);
1536 if (!IS_ERR(rt)) {
David S. Miller6700c272012-07-17 03:29:28 -07001537 ip_rt_update_pmtu(&rt->dst, NULL, skb, mtu);
David S. Miller36393392012-06-14 22:21:46 -07001538 ip_rt_put(rt);
1539 }
1540}
1541EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1542
1543void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1544{
1545 const struct inet_sock *inet = inet_sk(sk);
1546
1547 return ipv4_update_pmtu(skb, sock_net(sk), mtu,
1548 sk->sk_bound_dev_if, sk->sk_mark,
1549 inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
1550 inet_sk_flowi_flags(sk));
1551}
1552EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
David S. Millerf39925d2011-02-09 22:00:16 -08001553
David S. Millerb42597e2012-07-11 21:25:45 -07001554void ipv4_redirect(struct sk_buff *skb, struct net *net,
1555 int oif, u32 mark, u8 protocol, int flow_flags)
1556{
1557 const struct iphdr *iph = (const struct iphdr *)skb->data;
1558 struct flowi4 fl4;
1559 struct rtable *rt;
1560
1561 flowi4_init_output(&fl4, oif, mark, RT_TOS(iph->tos), RT_SCOPE_UNIVERSE,
1562 protocol, flow_flags, iph->daddr, iph->saddr, 0, 0);
1563 rt = __ip_route_output_key(net, &fl4);
1564 if (!IS_ERR(rt)) {
David S. Miller6700c272012-07-17 03:29:28 -07001565 ip_do_redirect(&rt->dst, NULL, skb);
David S. Millerb42597e2012-07-11 21:25:45 -07001566 ip_rt_put(rt);
1567 }
1568}
1569EXPORT_SYMBOL_GPL(ipv4_redirect);
1570
1571void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1572{
1573 const struct inet_sock *inet = inet_sk(sk);
1574
1575 return ipv4_redirect(skb, sock_net(sk), sk->sk_bound_dev_if,
1576 sk->sk_mark,
1577 inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
1578 inet_sk_flowi_flags(sk));
1579}
1580EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1581
David S. Millerefbc3682011-12-01 13:38:59 -05001582static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1583{
1584 struct rtable *rt = (struct rtable *) dst;
1585
1586 if (rt_is_expired(rt))
1587 return NULL;
Timo Teräsd11a4dc2010-03-18 23:20:20 +00001588 return dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589}
1590
1591static void ipv4_dst_destroy(struct dst_entry *dst)
1592{
1593 struct rtable *rt = (struct rtable *) dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594
David S. Miller62fa8a82011-01-26 20:51:05 -08001595 if (rt->fi) {
1596 fib_info_put(rt->fi);
1597 rt->fi = NULL;
1598 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599}
1600
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601
1602static void ipv4_link_failure(struct sk_buff *skb)
1603{
1604 struct rtable *rt;
1605
1606 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1607
Eric Dumazet511c3f92009-06-02 05:14:27 +00001608 rt = skb_rtable(skb);
David S. Miller59436342012-07-10 06:58:42 -07001609 if (rt)
1610 dst_set_expires(&rt->dst, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001611}
1612
1613static int ip_rt_bug(struct sk_buff *skb)
1614{
Joe Perches91df42b2012-05-15 14:11:54 +00001615 pr_debug("%s: %pI4 -> %pI4, %s\n",
1616 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1617 skb->dev ? skb->dev->name : "?");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618 kfree_skb(skb);
Dave Jonesc378a9c2011-05-21 07:16:42 +00001619 WARN_ON(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001620 return 0;
1621}
1622
1623/*
1624 We do not cache source address of outgoing interface,
1625 because it is used only by IP RR, TS and SRR options,
1626 so that it out of fast path.
1627
1628 BTW remember: "addr" is allowed to be not aligned
1629 in IP options!
1630 */
1631
David S. Miller8e363602011-05-13 17:29:41 -04001632void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633{
Al Viroa61ced52006-09-26 21:27:54 -07001634 __be32 src;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635
David S. Millerc7537962010-11-11 17:07:48 -08001636 if (rt_is_output_route(rt))
David S. Millerc5be24f2011-05-13 18:01:21 -04001637 src = ip_hdr(skb)->saddr;
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00001638 else {
David S. Miller8e363602011-05-13 17:29:41 -04001639 struct fib_result res;
1640 struct flowi4 fl4;
1641 struct iphdr *iph;
1642
1643 iph = ip_hdr(skb);
1644
1645 memset(&fl4, 0, sizeof(fl4));
1646 fl4.daddr = iph->daddr;
1647 fl4.saddr = iph->saddr;
Julian Anastasovb0fe4a32011-07-23 02:00:41 +00001648 fl4.flowi4_tos = RT_TOS(iph->tos);
David S. Miller8e363602011-05-13 17:29:41 -04001649 fl4.flowi4_oif = rt->dst.dev->ifindex;
1650 fl4.flowi4_iif = skb->dev->ifindex;
1651 fl4.flowi4_mark = skb->mark;
David S. Miller5e2b61f2011-03-04 21:47:09 -08001652
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00001653 rcu_read_lock();
David S. Miller68a5e3d2011-03-11 20:07:33 -05001654 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
David S. Miller436c3b62011-03-24 17:42:21 -07001655 src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00001656 else
1657 src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001658 RT_SCOPE_UNIVERSE);
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00001659 rcu_read_unlock();
1660 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001661 memcpy(addr, &src, 4);
1662}
1663
Patrick McHardyc7066f72011-01-14 13:36:42 +01001664#ifdef CONFIG_IP_ROUTE_CLASSID
Linus Torvalds1da177e2005-04-16 15:20:36 -07001665static void set_class_tag(struct rtable *rt, u32 tag)
1666{
Changli Gaod8d1f302010-06-10 23:31:35 -07001667 if (!(rt->dst.tclassid & 0xFFFF))
1668 rt->dst.tclassid |= tag & 0xFFFF;
1669 if (!(rt->dst.tclassid & 0xFFFF0000))
1670 rt->dst.tclassid |= tag & 0xFFFF0000;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671}
1672#endif
1673
David S. Miller0dbaee32010-12-13 12:52:14 -08001674static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1675{
1676 unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1677
1678 if (advmss == 0) {
1679 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1680 ip_rt_min_advmss);
1681 if (advmss > 65535 - 40)
1682 advmss = 65535 - 40;
1683 }
1684 return advmss;
1685}
1686
Steffen Klassertebb762f2011-11-23 02:12:51 +00001687static unsigned int ipv4_mtu(const struct dst_entry *dst)
David S. Millerd33e4552010-12-14 13:01:14 -08001688{
Steffen Klassert261663b2011-11-23 02:14:50 +00001689 const struct rtable *rt = (const struct rtable *) dst;
David S. Miller59436342012-07-10 06:58:42 -07001690 unsigned int mtu = rt->rt_pmtu;
1691
1692 if (mtu && time_after_eq(jiffies, rt->dst.expires))
1693 mtu = 0;
1694
1695 if (!mtu)
1696 mtu = dst_metric_raw(dst, RTAX_MTU);
Steffen Klassert618f9bc2011-11-23 02:13:31 +00001697
Steffen Klassert261663b2011-11-23 02:14:50 +00001698 if (mtu && rt_is_output_route(rt))
Steffen Klassert618f9bc2011-11-23 02:13:31 +00001699 return mtu;
1700
1701 mtu = dst->dev->mtu;
David S. Millerd33e4552010-12-14 13:01:14 -08001702
1703 if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
David S. Millerd33e4552010-12-14 13:01:14 -08001704
1705 if (rt->rt_gateway != rt->rt_dst && mtu > 576)
1706 mtu = 576;
1707 }
1708
1709 if (mtu > IP_MAX_MTU)
1710 mtu = IP_MAX_MTU;
1711
1712 return mtu;
1713}
1714
David S. Miller813b3b52011-04-28 14:48:42 -07001715static void rt_init_metrics(struct rtable *rt, const struct flowi4 *fl4,
David S. Miller5e2b61f2011-03-04 21:47:09 -08001716 struct fib_info *fi)
David S. Millera4daad62011-01-27 22:01:53 -08001717{
David S. Millerf1850712012-07-10 07:26:01 -07001718 if (fi->fib_metrics != (u32 *) dst_default_metrics) {
1719 rt->fi = fi;
1720 atomic_inc(&fi->fib_clntref);
David S. Millera4daad62011-01-27 22:01:53 -08001721 }
David S. Millerf1850712012-07-10 07:26:01 -07001722 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
David S. Millera4daad62011-01-27 22:01:53 -08001723}
1724
David S. Miller813b3b52011-04-28 14:48:42 -07001725static void rt_set_nexthop(struct rtable *rt, const struct flowi4 *fl4,
David S. Miller5e2b61f2011-03-04 21:47:09 -08001726 const struct fib_result *res,
David S. Miller982721f2011-02-16 21:44:24 -08001727 struct fib_info *fi, u16 type, u32 itag)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729 if (fi) {
1730 if (FIB_RES_GW(*res) &&
1731 FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
1732 rt->rt_gateway = FIB_RES_GW(*res);
David S. Miller813b3b52011-04-28 14:48:42 -07001733 rt_init_metrics(rt, fl4, fi);
Patrick McHardyc7066f72011-01-14 13:36:42 +01001734#ifdef CONFIG_IP_ROUTE_CLASSID
David S. Miller710ab6c2012-07-10 07:02:09 -07001735 rt->dst.tclassid = FIB_RES_NH(*res).nh_tclassid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736#endif
David S. Millerd33e4552010-12-14 13:01:14 -08001737 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738
Patrick McHardyc7066f72011-01-14 13:36:42 +01001739#ifdef CONFIG_IP_ROUTE_CLASSID
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740#ifdef CONFIG_IP_MULTIPLE_TABLES
David S. Miller85b91b02012-07-13 08:21:29 -07001741 set_class_tag(rt, res->tclassid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001742#endif
1743 set_class_tag(rt, itag);
1744#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001745}
1746
David S. Miller5c1e6aa2011-04-28 14:13:38 -07001747static struct rtable *rt_dst_alloc(struct net_device *dev,
1748 bool nopolicy, bool noxfrm)
David S. Miller0c4dcd52011-02-17 15:42:37 -08001749{
David S. Miller5c1e6aa2011-04-28 14:13:38 -07001750 return dst_alloc(&ipv4_dst_ops, dev, 1, -1,
1751 DST_HOST |
1752 (nopolicy ? DST_NOPOLICY : 0) |
1753 (noxfrm ? DST_NOXFRM : 0));
David S. Miller0c4dcd52011-02-17 15:42:37 -08001754}
1755
Eric Dumazet96d36222010-06-02 19:21:31 +00001756/* called in rcu_read_lock() section */
Al Viro9e12bb22006-09-26 21:25:20 -07001757static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758 u8 tos, struct net_device *dev, int our)
1759{
Eric Dumazet96d36222010-06-02 19:21:31 +00001760 unsigned int hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761 struct rtable *rth;
Eric Dumazet96d36222010-06-02 19:21:31 +00001762 struct in_device *in_dev = __in_dev_get_rcu(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001763 u32 itag = 0;
Eric Dumazetb5f7e752010-06-02 12:05:27 +00001764 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765
1766 /* Primary sanity checks. */
1767
1768 if (in_dev == NULL)
1769 return -EINVAL;
1770
Jan Engelhardt1e637c72008-01-21 03:18:08 -08001771 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
Thomas Grafd0daebc32012-06-12 00:44:01 +00001772 skb->protocol != htons(ETH_P_IP))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773 goto e_inval;
1774
Thomas Grafd0daebc32012-06-12 00:44:01 +00001775 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1776 if (ipv4_is_loopback(saddr))
1777 goto e_inval;
1778
Joe Perchesf97c1e02007-12-16 13:45:43 -08001779 if (ipv4_is_zeronet(saddr)) {
1780 if (!ipv4_is_local_multicast(daddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781 goto e_inval;
Eric Dumazetb5f7e752010-06-02 12:05:27 +00001782 } else {
David S. Miller9e56e382012-06-28 18:54:02 -07001783 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1784 in_dev, &itag);
Eric Dumazetb5f7e752010-06-02 12:05:27 +00001785 if (err < 0)
1786 goto e_err;
1787 }
Benjamin LaHaise4e7b2f12012-03-27 15:55:32 +00001788 rth = rt_dst_alloc(dev_net(dev)->loopback_dev,
David S. Miller5c1e6aa2011-04-28 14:13:38 -07001789 IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790 if (!rth)
1791 goto e_nobufs;
1792
Patrick McHardyc7066f72011-01-14 13:36:42 +01001793#ifdef CONFIG_IP_ROUTE_CLASSID
Changli Gaod8d1f302010-06-10 23:31:35 -07001794 rth->dst.tclassid = itag;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001795#endif
David S. Millercf911662011-04-28 14:31:47 -07001796 rth->dst.output = ip_rt_bug;
1797
1798 rth->rt_key_dst = daddr;
1799 rth->rt_key_src = saddr;
Denis V. Luneve84f84f2008-07-05 19:04:32 -07001800 rth->rt_genid = rt_genid(dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001801 rth->rt_flags = RTCF_MULTICAST;
Eric Dumazet29e75252008-01-31 17:05:09 -08001802 rth->rt_type = RTN_MULTICAST;
David S. Miller475949d2011-05-03 19:45:15 -07001803 rth->rt_key_tos = tos;
David S. Millercf911662011-04-28 14:31:47 -07001804 rth->rt_dst = daddr;
1805 rth->rt_src = saddr;
1806 rth->rt_route_iif = dev->ifindex;
1807 rth->rt_iif = dev->ifindex;
1808 rth->rt_oif = 0;
1809 rth->rt_mark = skb->mark;
David S. Miller59436342012-07-10 06:58:42 -07001810 rth->rt_pmtu = 0;
David S. Millercf911662011-04-28 14:31:47 -07001811 rth->rt_gateway = daddr;
David S. Millercf911662011-04-28 14:31:47 -07001812 rth->fi = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001813 if (our) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001814 rth->dst.input= ip_local_deliver;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815 rth->rt_flags |= RTCF_LOCAL;
1816 }
1817
1818#ifdef CONFIG_IP_MROUTE
Joe Perchesf97c1e02007-12-16 13:45:43 -08001819 if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
Changli Gaod8d1f302010-06-10 23:31:35 -07001820 rth->dst.input = ip_mr_input;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001821#endif
1822 RT_CACHE_STAT_INC(in_slow_mc);
1823
Denis V. Luneve84f84f2008-07-05 19:04:32 -07001824 hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
David S. Millerb23dd4f2011-03-02 14:31:35 -08001825 rth = rt_intern_hash(hash, rth, skb, dev->ifindex);
Eric Dumazet9aa3c942011-06-18 11:59:18 -07001826 return IS_ERR(rth) ? PTR_ERR(rth) : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001827
1828e_nobufs:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829 return -ENOBUFS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830e_inval:
Eric Dumazet96d36222010-06-02 19:21:31 +00001831 return -EINVAL;
Eric Dumazetb5f7e752010-06-02 12:05:27 +00001832e_err:
Eric Dumazetb5f7e752010-06-02 12:05:27 +00001833 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001834}
1835
1836
1837static void ip_handle_martian_source(struct net_device *dev,
1838 struct in_device *in_dev,
1839 struct sk_buff *skb,
Al Viro9e12bb22006-09-26 21:25:20 -07001840 __be32 daddr,
1841 __be32 saddr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001842{
1843 RT_CACHE_STAT_INC(in_martian_src);
1844#ifdef CONFIG_IP_ROUTE_VERBOSE
1845 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1846 /*
1847 * RFC1812 recommendation, if source is martian,
1848 * the only hint is MAC header.
1849 */
Joe Perches058bd4d2012-03-11 18:36:11 +00001850 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
Harvey Harrison673d57e2008-10-31 00:53:57 -07001851 &daddr, &saddr, dev->name);
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001852 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
Joe Perches058bd4d2012-03-11 18:36:11 +00001853 print_hex_dump(KERN_WARNING, "ll header: ",
1854 DUMP_PREFIX_OFFSET, 16, 1,
1855 skb_mac_header(skb),
1856 dev->hard_header_len, true);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001857 }
1858 }
1859#endif
1860}
1861
Eric Dumazet47360222010-06-03 04:13:21 +00001862/* called in rcu_read_lock() section */
Stephen Hemminger5969f712008-04-10 01:52:09 -07001863static int __mkroute_input(struct sk_buff *skb,
David S. Miller982721f2011-02-16 21:44:24 -08001864 const struct fib_result *res,
Stephen Hemminger5969f712008-04-10 01:52:09 -07001865 struct in_device *in_dev,
1866 __be32 daddr, __be32 saddr, u32 tos,
1867 struct rtable **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001868{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869 struct rtable *rth;
1870 int err;
1871 struct in_device *out_dev;
Eric Dumazet47360222010-06-03 04:13:21 +00001872 unsigned int flags = 0;
Al Virod9c9df82006-09-26 21:28:14 -07001873 u32 itag;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874
1875 /* get a working reference to the output device */
Eric Dumazet47360222010-06-03 04:13:21 +00001876 out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001877 if (out_dev == NULL) {
Joe Perchese87cc472012-05-13 21:56:26 +00001878 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001879 return -EINVAL;
1880 }
1881
1882
Michael Smith5c04c812011-04-07 04:51:50 +00001883 err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
David S. Miller9e56e382012-06-28 18:54:02 -07001884 in_dev->dev, in_dev, &itag);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001885 if (err < 0) {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001886 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887 saddr);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001888
Linus Torvalds1da177e2005-04-16 15:20:36 -07001889 goto cleanup;
1890 }
1891
1892 if (err)
1893 flags |= RTCF_DIRECTSRC;
1894
Thomas Graf51b77ca2008-06-03 16:36:01 -07001895 if (out_dev == in_dev && err &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001896 (IN_DEV_SHARED_MEDIA(out_dev) ||
1897 inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1898 flags |= RTCF_DOREDIRECT;
1899
1900 if (skb->protocol != htons(ETH_P_IP)) {
1901 /* Not IP (i.e. ARP). Do not create route, if it is
1902 * invalid for proxy arp. DNAT routes are always valid.
Jesper Dangaard Brouer65324142010-01-05 05:50:47 +00001903 *
1904 * Proxy arp feature have been extended to allow, ARP
1905 * replies back to the same interface, to support
1906 * Private VLAN switch technologies. See arp.c.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001907 */
Jesper Dangaard Brouer65324142010-01-05 05:50:47 +00001908 if (out_dev == in_dev &&
1909 IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001910 err = -EINVAL;
1911 goto cleanup;
1912 }
1913 }
1914
David S. Miller5c1e6aa2011-04-28 14:13:38 -07001915 rth = rt_dst_alloc(out_dev->dev,
1916 IN_DEV_CONF_GET(in_dev, NOPOLICY),
David S. Miller0c4dcd52011-02-17 15:42:37 -08001917 IN_DEV_CONF_GET(out_dev, NOXFRM));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918 if (!rth) {
1919 err = -ENOBUFS;
1920 goto cleanup;
1921 }
1922
David S. Miller5e2b61f2011-03-04 21:47:09 -08001923 rth->rt_key_dst = daddr;
David S. Miller5e2b61f2011-03-04 21:47:09 -08001924 rth->rt_key_src = saddr;
David S. Millercf911662011-04-28 14:31:47 -07001925 rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
1926 rth->rt_flags = flags;
1927 rth->rt_type = res->type;
David S. Miller475949d2011-05-03 19:45:15 -07001928 rth->rt_key_tos = tos;
David S. Millercf911662011-04-28 14:31:47 -07001929 rth->rt_dst = daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001930 rth->rt_src = saddr;
OGAWA Hirofumi1b86a582011-04-07 14:04:08 -07001931 rth->rt_route_iif = in_dev->dev->ifindex;
David S. Miller5e2b61f2011-03-04 21:47:09 -08001932 rth->rt_iif = in_dev->dev->ifindex;
David S. Miller5e2b61f2011-03-04 21:47:09 -08001933 rth->rt_oif = 0;
David S. Millercf911662011-04-28 14:31:47 -07001934 rth->rt_mark = skb->mark;
David S. Miller59436342012-07-10 06:58:42 -07001935 rth->rt_pmtu = 0;
David S. Millercf911662011-04-28 14:31:47 -07001936 rth->rt_gateway = daddr;
David S. Millercf911662011-04-28 14:31:47 -07001937 rth->fi = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001938
Changli Gaod8d1f302010-06-10 23:31:35 -07001939 rth->dst.input = ip_forward;
1940 rth->dst.output = ip_output;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001941
David S. Miller5e2b61f2011-03-04 21:47:09 -08001942 rt_set_nexthop(rth, NULL, res, res->fi, res->type, itag);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001943
Linus Torvalds1da177e2005-04-16 15:20:36 -07001944 *result = rth;
1945 err = 0;
1946 cleanup:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001947 return err;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001948}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001949
Stephen Hemminger5969f712008-04-10 01:52:09 -07001950static int ip_mkroute_input(struct sk_buff *skb,
1951 struct fib_result *res,
David S. Miller68a5e3d2011-03-11 20:07:33 -05001952 const struct flowi4 *fl4,
Stephen Hemminger5969f712008-04-10 01:52:09 -07001953 struct in_device *in_dev,
1954 __be32 daddr, __be32 saddr, u32 tos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001955{
Daniel Baluta5e73ea12012-04-15 01:34:41 +00001956 struct rtable *rth = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 int err;
Eric Dumazet95c96172012-04-15 05:58:06 +00001958 unsigned int hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001959
1960#ifdef CONFIG_IP_ROUTE_MULTIPATH
David S. Millerff3fccb2011-03-10 16:23:24 -08001961 if (res->fi && res->fi->fib_nhs > 1)
David S. Miller1b7fe5932011-03-10 17:01:16 -08001962 fib_select_multipath(res);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001963#endif
1964
1965 /* create a routing cache entry */
1966 err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
1967 if (err)
1968 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001969
1970 /* put it into the cache */
David S. Miller68a5e3d2011-03-11 20:07:33 -05001971 hash = rt_hash(daddr, saddr, fl4->flowi4_iif,
Changli Gaod8d1f302010-06-10 23:31:35 -07001972 rt_genid(dev_net(rth->dst.dev)));
David S. Miller68a5e3d2011-03-11 20:07:33 -05001973 rth = rt_intern_hash(hash, rth, skb, fl4->flowi4_iif);
David S. Millerb23dd4f2011-03-02 14:31:35 -08001974 if (IS_ERR(rth))
1975 return PTR_ERR(rth);
1976 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977}
1978
Linus Torvalds1da177e2005-04-16 15:20:36 -07001979/*
1980 * NOTE. We drop all the packets that has local source
1981 * addresses, because every properly looped back packet
1982 * must have correct destination already attached by output routine.
1983 *
1984 * Such approach solves two big problems:
1985 * 1. Not simplex devices are handled properly.
1986 * 2. IP spoofing attempts are filtered with 100% of guarantee.
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00001987 * called with rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07001988 */
1989
Al Viro9e12bb22006-09-26 21:25:20 -07001990static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
David S. Millerc10237e2012-06-27 17:05:06 -07001991 u8 tos, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001992{
1993 struct fib_result res;
Eric Dumazet96d36222010-06-02 19:21:31 +00001994 struct in_device *in_dev = __in_dev_get_rcu(dev);
David S. Miller68a5e3d2011-03-11 20:07:33 -05001995 struct flowi4 fl4;
Eric Dumazet95c96172012-04-15 05:58:06 +00001996 unsigned int flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001997 u32 itag = 0;
Eric Dumazet95c96172012-04-15 05:58:06 +00001998 struct rtable *rth;
1999 unsigned int hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002000 int err = -EINVAL;
Daniel Baluta5e73ea12012-04-15 01:34:41 +00002001 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002002
2003 /* IP on this device is disabled. */
2004
2005 if (!in_dev)
2006 goto out;
2007
2008 /* Check for the most weird martians, which can be not detected
2009 by fib_lookup.
2010 */
2011
Thomas Grafd0daebc32012-06-12 00:44:01 +00002012 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002013 goto martian_source;
2014
Andy Walls27a954b2010-10-17 15:11:22 +00002015 if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002016 goto brd_input;
2017
2018 /* Accept zero addresses only to limited broadcast;
2019 * I even do not know to fix it or not. Waiting for complains :-)
2020 */
Joe Perchesf97c1e02007-12-16 13:45:43 -08002021 if (ipv4_is_zeronet(saddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002022 goto martian_source;
2023
Thomas Grafd0daebc32012-06-12 00:44:01 +00002024 if (ipv4_is_zeronet(daddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002025 goto martian_destination;
2026
Thomas Grafd0daebc32012-06-12 00:44:01 +00002027 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev))) {
2028 if (ipv4_is_loopback(daddr))
2029 goto martian_destination;
2030
2031 if (ipv4_is_loopback(saddr))
2032 goto martian_source;
2033 }
2034
Linus Torvalds1da177e2005-04-16 15:20:36 -07002035 /*
2036 * Now we are ready to route packet.
2037 */
David S. Miller68a5e3d2011-03-11 20:07:33 -05002038 fl4.flowi4_oif = 0;
2039 fl4.flowi4_iif = dev->ifindex;
2040 fl4.flowi4_mark = skb->mark;
2041 fl4.flowi4_tos = tos;
2042 fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
2043 fl4.daddr = daddr;
2044 fl4.saddr = saddr;
2045 err = fib_lookup(net, &fl4, &res);
David S. Miller251da412012-06-26 16:27:09 -07002046 if (err != 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002047 goto no_route;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002048
2049 RT_CACHE_STAT_INC(in_slow_tot);
2050
2051 if (res.type == RTN_BROADCAST)
2052 goto brd_input;
2053
2054 if (res.type == RTN_LOCAL) {
Michael Smith5c04c812011-04-07 04:51:50 +00002055 err = fib_validate_source(skb, saddr, daddr, tos,
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002056 net->loopback_dev->ifindex,
David S. Miller9e56e382012-06-28 18:54:02 -07002057 dev, in_dev, &itag);
Eric Dumazetb5f7e752010-06-02 12:05:27 +00002058 if (err < 0)
2059 goto martian_source_keep_err;
2060 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002061 flags |= RTCF_DIRECTSRC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002062 goto local_input;
2063 }
2064
2065 if (!IN_DEV_FORWARD(in_dev))
David S. Miller251da412012-06-26 16:27:09 -07002066 goto no_route;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002067 if (res.type != RTN_UNICAST)
2068 goto martian_destination;
2069
David S. Miller68a5e3d2011-03-11 20:07:33 -05002070 err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002071out: return err;
2072
2073brd_input:
2074 if (skb->protocol != htons(ETH_P_IP))
2075 goto e_inval;
2076
David S. Miller41347dc2012-06-28 04:05:27 -07002077 if (!ipv4_is_zeronet(saddr)) {
David S. Miller9e56e382012-06-28 18:54:02 -07002078 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2079 in_dev, &itag);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080 if (err < 0)
Eric Dumazetb5f7e752010-06-02 12:05:27 +00002081 goto martian_source_keep_err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002082 if (err)
2083 flags |= RTCF_DIRECTSRC;
2084 }
2085 flags |= RTCF_BROADCAST;
2086 res.type = RTN_BROADCAST;
2087 RT_CACHE_STAT_INC(in_brd);
2088
2089local_input:
David S. Miller5c1e6aa2011-04-28 14:13:38 -07002090 rth = rt_dst_alloc(net->loopback_dev,
2091 IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002092 if (!rth)
2093 goto e_nobufs;
2094
David S. Millercf911662011-04-28 14:31:47 -07002095 rth->dst.input= ip_local_deliver;
Changli Gaod8d1f302010-06-10 23:31:35 -07002096 rth->dst.output= ip_rt_bug;
David S. Millercf911662011-04-28 14:31:47 -07002097#ifdef CONFIG_IP_ROUTE_CLASSID
2098 rth->dst.tclassid = itag;
2099#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002100
David S. Miller5e2b61f2011-03-04 21:47:09 -08002101 rth->rt_key_dst = daddr;
David S. Miller5e2b61f2011-03-04 21:47:09 -08002102 rth->rt_key_src = saddr;
David S. Millercf911662011-04-28 14:31:47 -07002103 rth->rt_genid = rt_genid(net);
2104 rth->rt_flags = flags|RTCF_LOCAL;
2105 rth->rt_type = res.type;
David S. Miller475949d2011-05-03 19:45:15 -07002106 rth->rt_key_tos = tos;
David S. Millercf911662011-04-28 14:31:47 -07002107 rth->rt_dst = daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002108 rth->rt_src = saddr;
OGAWA Hirofumi1b86a582011-04-07 14:04:08 -07002109 rth->rt_route_iif = dev->ifindex;
David S. Miller5e2b61f2011-03-04 21:47:09 -08002110 rth->rt_iif = dev->ifindex;
David S. Millercf911662011-04-28 14:31:47 -07002111 rth->rt_oif = 0;
2112 rth->rt_mark = skb->mark;
David S. Miller59436342012-07-10 06:58:42 -07002113 rth->rt_pmtu = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002114 rth->rt_gateway = daddr;
David S. Millercf911662011-04-28 14:31:47 -07002115 rth->fi = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002116 if (res.type == RTN_UNREACHABLE) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002117 rth->dst.input= ip_error;
2118 rth->dst.error= -err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002119 rth->rt_flags &= ~RTCF_LOCAL;
2120 }
David S. Miller68a5e3d2011-03-11 20:07:33 -05002121 hash = rt_hash(daddr, saddr, fl4.flowi4_iif, rt_genid(net));
2122 rth = rt_intern_hash(hash, rth, skb, fl4.flowi4_iif);
David S. Millerb23dd4f2011-03-02 14:31:35 -08002123 err = 0;
2124 if (IS_ERR(rth))
2125 err = PTR_ERR(rth);
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002126 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002127
2128no_route:
2129 RT_CACHE_STAT_INC(in_no_route);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002130 res.type = RTN_UNREACHABLE;
Mitsuru Chinen7f538782007-12-07 01:07:24 -08002131 if (err == -ESRCH)
2132 err = -ENETUNREACH;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002133 goto local_input;
2134
2135 /*
2136 * Do not cache martian addresses: they should be logged (RFC1812)
2137 */
2138martian_destination:
2139 RT_CACHE_STAT_INC(in_martian_dst);
2140#ifdef CONFIG_IP_ROUTE_VERBOSE
Joe Perchese87cc472012-05-13 21:56:26 +00002141 if (IN_DEV_LOG_MARTIANS(in_dev))
2142 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2143 &daddr, &saddr, dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002144#endif
Dietmar Eggemann2c2910a2005-06-28 13:06:23 -07002145
Linus Torvalds1da177e2005-04-16 15:20:36 -07002146e_inval:
2147 err = -EINVAL;
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002148 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002149
2150e_nobufs:
2151 err = -ENOBUFS;
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002152 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002153
2154martian_source:
Eric Dumazetb5f7e752010-06-02 12:05:27 +00002155 err = -EINVAL;
2156martian_source_keep_err:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002157 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002158 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002159}
2160
Eric Dumazet407eadd2010-05-10 11:32:55 +00002161int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
David S. Millerc10237e2012-06-27 17:05:06 -07002162 u8 tos, struct net_device *dev, bool noref)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002163{
Eric Dumazet95c96172012-04-15 05:58:06 +00002164 struct rtable *rth;
2165 unsigned int hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002166 int iif = dev->ifindex;
Denis V. Lunevb5921912008-01-22 23:50:25 -08002167 struct net *net;
Eric Dumazet96d36222010-06-02 19:21:31 +00002168 int res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002169
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09002170 net = dev_net(dev);
Neil Horman1080d702008-10-27 12:28:25 -07002171
Eric Dumazet96d36222010-06-02 19:21:31 +00002172 rcu_read_lock();
2173
Neil Horman1080d702008-10-27 12:28:25 -07002174 if (!rt_caching(net))
2175 goto skip_cache;
2176
Linus Torvalds1da177e2005-04-16 15:20:36 -07002177 tos &= IPTOS_RT_MASK;
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002178 hash = rt_hash(daddr, saddr, iif, rt_genid(net));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002179
Linus Torvalds1da177e2005-04-16 15:20:36 -07002180 for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
Changli Gaod8d1f302010-06-10 23:31:35 -07002181 rth = rcu_dereference(rth->dst.rt_next)) {
David S. Miller5e2b61f2011-03-04 21:47:09 -08002182 if ((((__force u32)rth->rt_key_dst ^ (__force u32)daddr) |
2183 ((__force u32)rth->rt_key_src ^ (__force u32)saddr) |
Julian Anastasov97a80412011-08-09 04:01:16 +00002184 (rth->rt_route_iif ^ iif) |
David S. Miller475949d2011-05-03 19:45:15 -07002185 (rth->rt_key_tos ^ tos)) == 0 &&
David S. Miller5e2b61f2011-03-04 21:47:09 -08002186 rth->rt_mark == skb->mark &&
Changli Gaod8d1f302010-06-10 23:31:35 -07002187 net_eq(dev_net(rth->dst.dev), net) &&
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002188 !rt_is_expired(rth)) {
Eric Dumazet407eadd2010-05-10 11:32:55 +00002189 if (noref) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002190 dst_use_noref(&rth->dst, jiffies);
2191 skb_dst_set_noref(skb, &rth->dst);
Eric Dumazet407eadd2010-05-10 11:32:55 +00002192 } else {
Changli Gaod8d1f302010-06-10 23:31:35 -07002193 dst_use(&rth->dst, jiffies);
2194 skb_dst_set(skb, &rth->dst);
Eric Dumazet407eadd2010-05-10 11:32:55 +00002195 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002196 RT_CACHE_STAT_INC(in_hit);
2197 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002198 return 0;
2199 }
2200 RT_CACHE_STAT_INC(in_hlist_search);
2201 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002202
Neil Horman1080d702008-10-27 12:28:25 -07002203skip_cache:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002204 /* Multicast recognition logic is moved from route cache to here.
2205 The problem was that too many Ethernet cards have broken/missing
2206 hardware multicast filters :-( As result the host on multicasting
2207 network acquires a lot of useless route cache entries, sort of
2208 SDR messages from all the world. Now we try to get rid of them.
2209 Really, provided software IP multicast filter is organized
2210 reasonably (at least, hashed), it does not result in a slowdown
2211 comparing with route cache reject entries.
2212 Note, that multicast routers are not affected, because
2213 route cache entry is created eventually.
2214 */
Joe Perchesf97c1e02007-12-16 13:45:43 -08002215 if (ipv4_is_multicast(daddr)) {
Eric Dumazet96d36222010-06-02 19:21:31 +00002216 struct in_device *in_dev = __in_dev_get_rcu(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002217
Eric Dumazet96d36222010-06-02 19:21:31 +00002218 if (in_dev) {
David S. Millerdbdd9a52011-03-10 16:34:38 -08002219 int our = ip_check_mc_rcu(in_dev, daddr, saddr,
2220 ip_hdr(skb)->protocol);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002221 if (our
2222#ifdef CONFIG_IP_MROUTE
Joe Perches9d4fb272009-11-23 10:41:23 -08002223 ||
2224 (!ipv4_is_local_multicast(daddr) &&
2225 IN_DEV_MFORWARD(in_dev))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002226#endif
Joe Perches9d4fb272009-11-23 10:41:23 -08002227 ) {
Eric Dumazet96d36222010-06-02 19:21:31 +00002228 int res = ip_route_input_mc(skb, daddr, saddr,
2229 tos, dev, our);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002230 rcu_read_unlock();
Eric Dumazet96d36222010-06-02 19:21:31 +00002231 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002232 }
2233 }
2234 rcu_read_unlock();
2235 return -EINVAL;
2236 }
David S. Millerc10237e2012-06-27 17:05:06 -07002237 res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
Eric Dumazet96d36222010-06-02 19:21:31 +00002238 rcu_read_unlock();
2239 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002240}
Eric Dumazet407eadd2010-05-10 11:32:55 +00002241EXPORT_SYMBOL(ip_route_input_common);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002242
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002243/* called with rcu_read_lock() */
David S. Miller982721f2011-02-16 21:44:24 -08002244static struct rtable *__mkroute_output(const struct fib_result *res,
David S. Miller68a5e3d2011-03-11 20:07:33 -05002245 const struct flowi4 *fl4,
David S. Miller813b3b52011-04-28 14:48:42 -07002246 __be32 orig_daddr, __be32 orig_saddr,
Julian Anastasovf61759e2011-12-02 11:39:42 +00002247 int orig_oif, __u8 orig_rtos,
2248 struct net_device *dev_out,
David S. Miller5ada5522011-02-17 15:29:00 -08002249 unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002250{
David S. Miller982721f2011-02-16 21:44:24 -08002251 struct fib_info *fi = res->fi;
David S. Miller5ada5522011-02-17 15:29:00 -08002252 struct in_device *in_dev;
David S. Miller982721f2011-02-16 21:44:24 -08002253 u16 type = res->type;
David S. Miller5ada5522011-02-17 15:29:00 -08002254 struct rtable *rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255
Thomas Grafd0daebc32012-06-12 00:44:01 +00002256 in_dev = __in_dev_get_rcu(dev_out);
2257 if (!in_dev)
David S. Miller5ada5522011-02-17 15:29:00 -08002258 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002259
Thomas Grafd0daebc32012-06-12 00:44:01 +00002260 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2261 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
2262 return ERR_PTR(-EINVAL);
2263
David S. Miller68a5e3d2011-03-11 20:07:33 -05002264 if (ipv4_is_lbcast(fl4->daddr))
David S. Miller982721f2011-02-16 21:44:24 -08002265 type = RTN_BROADCAST;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002266 else if (ipv4_is_multicast(fl4->daddr))
David S. Miller982721f2011-02-16 21:44:24 -08002267 type = RTN_MULTICAST;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002268 else if (ipv4_is_zeronet(fl4->daddr))
David S. Miller5ada5522011-02-17 15:29:00 -08002269 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002270
2271 if (dev_out->flags & IFF_LOOPBACK)
2272 flags |= RTCF_LOCAL;
2273
David S. Miller982721f2011-02-16 21:44:24 -08002274 if (type == RTN_BROADCAST) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002275 flags |= RTCF_BROADCAST | RTCF_LOCAL;
David S. Miller982721f2011-02-16 21:44:24 -08002276 fi = NULL;
2277 } else if (type == RTN_MULTICAST) {
Eric Dumazetdd28d1a2010-09-29 11:53:50 +00002278 flags |= RTCF_MULTICAST | RTCF_LOCAL;
David S. Miller813b3b52011-04-28 14:48:42 -07002279 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2280 fl4->flowi4_proto))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002281 flags &= ~RTCF_LOCAL;
2282 /* If multicast route do not exist use
Eric Dumazetdd28d1a2010-09-29 11:53:50 +00002283 * default one, but do not gateway in this case.
2284 * Yes, it is hack.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002285 */
David S. Miller982721f2011-02-16 21:44:24 -08002286 if (fi && res->prefixlen < 4)
2287 fi = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002288 }
2289
David S. Miller5c1e6aa2011-04-28 14:13:38 -07002290 rth = rt_dst_alloc(dev_out,
2291 IN_DEV_CONF_GET(in_dev, NOPOLICY),
David S. Miller0c4dcd52011-02-17 15:42:37 -08002292 IN_DEV_CONF_GET(in_dev, NOXFRM));
Dimitris Michailidis8391d072010-10-07 14:48:38 +00002293 if (!rth)
David S. Miller5ada5522011-02-17 15:29:00 -08002294 return ERR_PTR(-ENOBUFS);
Dimitris Michailidis8391d072010-10-07 14:48:38 +00002295
David S. Millercf911662011-04-28 14:31:47 -07002296 rth->dst.output = ip_output;
2297
David S. Miller813b3b52011-04-28 14:48:42 -07002298 rth->rt_key_dst = orig_daddr;
2299 rth->rt_key_src = orig_saddr;
David S. Millercf911662011-04-28 14:31:47 -07002300 rth->rt_genid = rt_genid(dev_net(dev_out));
2301 rth->rt_flags = flags;
2302 rth->rt_type = type;
Julian Anastasovf61759e2011-12-02 11:39:42 +00002303 rth->rt_key_tos = orig_rtos;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002304 rth->rt_dst = fl4->daddr;
2305 rth->rt_src = fl4->saddr;
OGAWA Hirofumi1b86a582011-04-07 14:04:08 -07002306 rth->rt_route_iif = 0;
David S. Miller813b3b52011-04-28 14:48:42 -07002307 rth->rt_iif = orig_oif ? : dev_out->ifindex;
2308 rth->rt_oif = orig_oif;
2309 rth->rt_mark = fl4->flowi4_mark;
David S. Miller59436342012-07-10 06:58:42 -07002310 rth->rt_pmtu = 0;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002311 rth->rt_gateway = fl4->daddr;
David S. Millercf911662011-04-28 14:31:47 -07002312 rth->fi = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002313
2314 RT_CACHE_STAT_INC(out_slow_tot);
2315
David S. Miller41347dc2012-06-28 04:05:27 -07002316 if (flags & RTCF_LOCAL)
Changli Gaod8d1f302010-06-10 23:31:35 -07002317 rth->dst.input = ip_local_deliver;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002318 if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002319 if (flags & RTCF_LOCAL &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07002320 !(dev_out->flags & IFF_LOOPBACK)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002321 rth->dst.output = ip_mc_output;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002322 RT_CACHE_STAT_INC(out_slow_mc);
2323 }
2324#ifdef CONFIG_IP_MROUTE
David S. Miller982721f2011-02-16 21:44:24 -08002325 if (type == RTN_MULTICAST) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002326 if (IN_DEV_MFORWARD(in_dev) &&
David S. Miller813b3b52011-04-28 14:48:42 -07002327 !ipv4_is_local_multicast(fl4->daddr)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002328 rth->dst.input = ip_mr_input;
2329 rth->dst.output = ip_mc_output;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002330 }
2331 }
2332#endif
2333 }
2334
David S. Miller813b3b52011-04-28 14:48:42 -07002335 rt_set_nexthop(rth, fl4, res, fi, type, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002336
Eric Dumazet7586ece2012-06-20 05:02:19 +00002337 if (fl4->flowi4_flags & FLOWI_FLAG_RT_NOCACHE)
2338 rth->dst.flags |= DST_NOCACHE;
2339
David S. Miller5ada5522011-02-17 15:29:00 -08002340 return rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002341}
2342
Linus Torvalds1da177e2005-04-16 15:20:36 -07002343/*
2344 * Major route resolver routine.
Eric Dumazet0197aa32010-09-30 03:33:58 +00002345 * called with rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002346 */
2347
David S. Miller813b3b52011-04-28 14:48:42 -07002348static struct rtable *ip_route_output_slow(struct net *net, struct flowi4 *fl4)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002349{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002350 struct net_device *dev_out = NULL;
Julian Anastasovf61759e2011-12-02 11:39:42 +00002351 __u8 tos = RT_FL_TOS(fl4);
David S. Miller813b3b52011-04-28 14:48:42 -07002352 unsigned int flags = 0;
2353 struct fib_result res;
David S. Miller5ada5522011-02-17 15:29:00 -08002354 struct rtable *rth;
David S. Miller813b3b52011-04-28 14:48:42 -07002355 __be32 orig_daddr;
2356 __be32 orig_saddr;
2357 int orig_oif;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002358
David S. Miller85b91b02012-07-13 08:21:29 -07002359 res.tclassid = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002360 res.fi = NULL;
David S. Miller8b96d222012-06-11 02:01:56 -07002361 res.table = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002362
David S. Miller813b3b52011-04-28 14:48:42 -07002363 orig_daddr = fl4->daddr;
2364 orig_saddr = fl4->saddr;
2365 orig_oif = fl4->flowi4_oif;
2366
2367 fl4->flowi4_iif = net->loopback_dev->ifindex;
2368 fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2369 fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2370 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
David S. Miller44713b62011-03-04 21:24:47 -08002371
David S. Miller010c2702011-02-17 15:37:09 -08002372 rcu_read_lock();
David S. Miller813b3b52011-04-28 14:48:42 -07002373 if (fl4->saddr) {
David S. Millerb23dd4f2011-03-02 14:31:35 -08002374 rth = ERR_PTR(-EINVAL);
David S. Miller813b3b52011-04-28 14:48:42 -07002375 if (ipv4_is_multicast(fl4->saddr) ||
2376 ipv4_is_lbcast(fl4->saddr) ||
2377 ipv4_is_zeronet(fl4->saddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002378 goto out;
2379
Linus Torvalds1da177e2005-04-16 15:20:36 -07002380 /* I removed check for oif == dev_out->oif here.
2381 It was wrong for two reasons:
Denis V. Lunev1ab35272008-01-22 22:04:30 -08002382 1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2383 is assigned to multiple interfaces.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002384 2. Moreover, we are allowed to send packets with saddr
2385 of another iface. --ANK
2386 */
2387
David S. Miller813b3b52011-04-28 14:48:42 -07002388 if (fl4->flowi4_oif == 0 &&
2389 (ipv4_is_multicast(fl4->daddr) ||
2390 ipv4_is_lbcast(fl4->daddr))) {
Julian Anastasova210d012008-10-01 07:28:28 -07002391 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
David S. Miller813b3b52011-04-28 14:48:42 -07002392 dev_out = __ip_dev_find(net, fl4->saddr, false);
Julian Anastasova210d012008-10-01 07:28:28 -07002393 if (dev_out == NULL)
2394 goto out;
2395
Linus Torvalds1da177e2005-04-16 15:20:36 -07002396 /* Special hack: user can direct multicasts
2397 and limited broadcast via necessary interface
2398 without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2399 This hack is not just for fun, it allows
2400 vic,vat and friends to work.
2401 They bind socket to loopback, set ttl to zero
2402 and expect that it will work.
2403 From the viewpoint of routing cache they are broken,
2404 because we are not allowed to build multicast path
2405 with loopback source addr (look, routing cache
2406 cannot know, that ttl is zero, so that packet
2407 will not leave this host and route is valid).
2408 Luckily, this hack is good workaround.
2409 */
2410
David S. Miller813b3b52011-04-28 14:48:42 -07002411 fl4->flowi4_oif = dev_out->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002412 goto make_route;
2413 }
Julian Anastasova210d012008-10-01 07:28:28 -07002414
David S. Miller813b3b52011-04-28 14:48:42 -07002415 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
Julian Anastasova210d012008-10-01 07:28:28 -07002416 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
David S. Miller813b3b52011-04-28 14:48:42 -07002417 if (!__ip_dev_find(net, fl4->saddr, false))
Julian Anastasova210d012008-10-01 07:28:28 -07002418 goto out;
Julian Anastasova210d012008-10-01 07:28:28 -07002419 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002420 }
2421
2422
David S. Miller813b3b52011-04-28 14:48:42 -07002423 if (fl4->flowi4_oif) {
2424 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
David S. Millerb23dd4f2011-03-02 14:31:35 -08002425 rth = ERR_PTR(-ENODEV);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002426 if (dev_out == NULL)
2427 goto out;
Herbert Xue5ed6392005-10-03 14:35:55 -07002428
2429 /* RACE: Check return value of inet_select_addr instead. */
Eric Dumazetfc75fc82010-12-22 04:39:39 +00002430 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
David S. Millerb23dd4f2011-03-02 14:31:35 -08002431 rth = ERR_PTR(-ENETUNREACH);
Eric Dumazetfc75fc82010-12-22 04:39:39 +00002432 goto out;
2433 }
David S. Miller813b3b52011-04-28 14:48:42 -07002434 if (ipv4_is_local_multicast(fl4->daddr) ||
2435 ipv4_is_lbcast(fl4->daddr)) {
2436 if (!fl4->saddr)
2437 fl4->saddr = inet_select_addr(dev_out, 0,
2438 RT_SCOPE_LINK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002439 goto make_route;
2440 }
David S. Miller813b3b52011-04-28 14:48:42 -07002441 if (fl4->saddr) {
2442 if (ipv4_is_multicast(fl4->daddr))
2443 fl4->saddr = inet_select_addr(dev_out, 0,
2444 fl4->flowi4_scope);
2445 else if (!fl4->daddr)
2446 fl4->saddr = inet_select_addr(dev_out, 0,
2447 RT_SCOPE_HOST);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002448 }
2449 }
2450
David S. Miller813b3b52011-04-28 14:48:42 -07002451 if (!fl4->daddr) {
2452 fl4->daddr = fl4->saddr;
2453 if (!fl4->daddr)
2454 fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
Denis V. Lunevb40afd02008-01-22 22:06:19 -08002455 dev_out = net->loopback_dev;
David S. Miller813b3b52011-04-28 14:48:42 -07002456 fl4->flowi4_oif = net->loopback_dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002457 res.type = RTN_LOCAL;
2458 flags |= RTCF_LOCAL;
2459 goto make_route;
2460 }
2461
David S. Miller813b3b52011-04-28 14:48:42 -07002462 if (fib_lookup(net, fl4, &res)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002463 res.fi = NULL;
David S. Miller8b96d222012-06-11 02:01:56 -07002464 res.table = NULL;
David S. Miller813b3b52011-04-28 14:48:42 -07002465 if (fl4->flowi4_oif) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002466 /* Apparently, routing tables are wrong. Assume,
2467 that the destination is on link.
2468
2469 WHY? DW.
2470 Because we are allowed to send to iface
2471 even if it has NO routes and NO assigned
2472 addresses. When oif is specified, routing
2473 tables are looked up with only one purpose:
2474 to catch if destination is gatewayed, rather than
2475 direct. Moreover, if MSG_DONTROUTE is set,
2476 we send packet, ignoring both routing tables
2477 and ifaddr state. --ANK
2478
2479
2480 We could make it even if oif is unknown,
2481 likely IPv6, but we do not.
2482 */
2483
David S. Miller813b3b52011-04-28 14:48:42 -07002484 if (fl4->saddr == 0)
2485 fl4->saddr = inet_select_addr(dev_out, 0,
2486 RT_SCOPE_LINK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002487 res.type = RTN_UNICAST;
2488 goto make_route;
2489 }
David S. Millerb23dd4f2011-03-02 14:31:35 -08002490 rth = ERR_PTR(-ENETUNREACH);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002491 goto out;
2492 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002493
2494 if (res.type == RTN_LOCAL) {
David S. Miller813b3b52011-04-28 14:48:42 -07002495 if (!fl4->saddr) {
Joel Sing9fc3bbb2011-01-03 20:24:20 +00002496 if (res.fi->fib_prefsrc)
David S. Miller813b3b52011-04-28 14:48:42 -07002497 fl4->saddr = res.fi->fib_prefsrc;
Joel Sing9fc3bbb2011-01-03 20:24:20 +00002498 else
David S. Miller813b3b52011-04-28 14:48:42 -07002499 fl4->saddr = fl4->daddr;
Joel Sing9fc3bbb2011-01-03 20:24:20 +00002500 }
Denis V. Lunevb40afd02008-01-22 22:06:19 -08002501 dev_out = net->loopback_dev;
David S. Miller813b3b52011-04-28 14:48:42 -07002502 fl4->flowi4_oif = dev_out->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002503 res.fi = NULL;
2504 flags |= RTCF_LOCAL;
2505 goto make_route;
2506 }
2507
2508#ifdef CONFIG_IP_ROUTE_MULTIPATH
David S. Miller813b3b52011-04-28 14:48:42 -07002509 if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
David S. Miller1b7fe5932011-03-10 17:01:16 -08002510 fib_select_multipath(&res);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002511 else
2512#endif
David S. Miller21d8c492011-04-14 14:49:37 -07002513 if (!res.prefixlen &&
2514 res.table->tb_num_default > 1 &&
David S. Miller813b3b52011-04-28 14:48:42 -07002515 res.type == RTN_UNICAST && !fl4->flowi4_oif)
David S. Miller0c838ff2011-01-31 16:16:50 -08002516 fib_select_default(&res);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002517
David S. Miller813b3b52011-04-28 14:48:42 -07002518 if (!fl4->saddr)
2519 fl4->saddr = FIB_RES_PREFSRC(net, res);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002520
Linus Torvalds1da177e2005-04-16 15:20:36 -07002521 dev_out = FIB_RES_DEV(res);
David S. Miller813b3b52011-04-28 14:48:42 -07002522 fl4->flowi4_oif = dev_out->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002523
2524
2525make_route:
David S. Miller813b3b52011-04-28 14:48:42 -07002526 rth = __mkroute_output(&res, fl4, orig_daddr, orig_saddr, orig_oif,
Julian Anastasovf61759e2011-12-02 11:39:42 +00002527 tos, dev_out, flags);
David S. Millerb23dd4f2011-03-02 14:31:35 -08002528 if (!IS_ERR(rth)) {
David S. Miller5ada5522011-02-17 15:29:00 -08002529 unsigned int hash;
2530
David S. Miller813b3b52011-04-28 14:48:42 -07002531 hash = rt_hash(orig_daddr, orig_saddr, orig_oif,
David S. Miller5ada5522011-02-17 15:29:00 -08002532 rt_genid(dev_net(dev_out)));
David S. Miller813b3b52011-04-28 14:48:42 -07002533 rth = rt_intern_hash(hash, rth, NULL, orig_oif);
David S. Miller5ada5522011-02-17 15:29:00 -08002534 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002535
David S. Miller010c2702011-02-17 15:37:09 -08002536out:
2537 rcu_read_unlock();
David S. Millerb23dd4f2011-03-02 14:31:35 -08002538 return rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002539}
2540
David S. Miller813b3b52011-04-28 14:48:42 -07002541struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *flp4)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002542{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002543 struct rtable *rth;
David S. Miller010c2702011-02-17 15:37:09 -08002544 unsigned int hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002545
Neil Horman1080d702008-10-27 12:28:25 -07002546 if (!rt_caching(net))
2547 goto slow_output;
2548
David S. Miller9d6ec932011-03-12 01:12:47 -05002549 hash = rt_hash(flp4->daddr, flp4->saddr, flp4->flowi4_oif, rt_genid(net));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002550
2551 rcu_read_lock_bh();
Paul E. McKenneya898def2010-02-22 17:04:49 -08002552 for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
Changli Gaod8d1f302010-06-10 23:31:35 -07002553 rth = rcu_dereference_bh(rth->dst.rt_next)) {
David S. Miller9d6ec932011-03-12 01:12:47 -05002554 if (rth->rt_key_dst == flp4->daddr &&
2555 rth->rt_key_src == flp4->saddr &&
David S. Millerc7537962010-11-11 17:07:48 -08002556 rt_is_output_route(rth) &&
David S. Miller9d6ec932011-03-12 01:12:47 -05002557 rth->rt_oif == flp4->flowi4_oif &&
2558 rth->rt_mark == flp4->flowi4_mark &&
David S. Miller475949d2011-05-03 19:45:15 -07002559 !((rth->rt_key_tos ^ flp4->flowi4_tos) &
Denis V. Lunevb5921912008-01-22 23:50:25 -08002560 (IPTOS_RT_MASK | RTO_ONLINK)) &&
Changli Gaod8d1f302010-06-10 23:31:35 -07002561 net_eq(dev_net(rth->dst.dev), net) &&
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002562 !rt_is_expired(rth)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002563 dst_use(&rth->dst, jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002564 RT_CACHE_STAT_INC(out_hit);
2565 rcu_read_unlock_bh();
David S. Miller56157872011-05-02 14:37:45 -07002566 if (!flp4->saddr)
2567 flp4->saddr = rth->rt_src;
2568 if (!flp4->daddr)
2569 flp4->daddr = rth->rt_dst;
David S. Millerb23dd4f2011-03-02 14:31:35 -08002570 return rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002571 }
2572 RT_CACHE_STAT_INC(out_hlist_search);
2573 }
2574 rcu_read_unlock_bh();
2575
Neil Horman1080d702008-10-27 12:28:25 -07002576slow_output:
David S. Miller9d6ec932011-03-12 01:12:47 -05002577 return ip_route_output_slow(net, flp4);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002578}
Arnaldo Carvalho de Melod8c97a92005-08-09 20:12:12 -07002579EXPORT_SYMBOL_GPL(__ip_route_output_key);
2580
Jianzhao Wangae2688d2010-09-08 14:35:43 -07002581static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2582{
2583 return NULL;
2584}
2585
Steffen Klassertebb762f2011-11-23 02:12:51 +00002586static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
Roland Dreierec831ea2011-01-31 13:16:00 -08002587{
Steffen Klassert618f9bc2011-11-23 02:13:31 +00002588 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2589
2590 return mtu ? : dst->dev->mtu;
Roland Dreierec831ea2011-01-31 13:16:00 -08002591}
2592
David S. Miller6700c272012-07-17 03:29:28 -07002593static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2594 struct sk_buff *skb, u32 mtu)
David S. Miller14e50e52007-05-24 18:17:54 -07002595{
2596}
2597
David S. Miller6700c272012-07-17 03:29:28 -07002598static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2599 struct sk_buff *skb)
David S. Millerb587ee32012-07-12 00:39:24 -07002600{
2601}
2602
Held Bernhard0972ddb2011-04-24 22:07:32 +00002603static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2604 unsigned long old)
2605{
2606 return NULL;
2607}
2608
David S. Miller14e50e52007-05-24 18:17:54 -07002609static struct dst_ops ipv4_dst_blackhole_ops = {
2610 .family = AF_INET,
Harvey Harrison09640e62009-02-01 00:45:17 -08002611 .protocol = cpu_to_be16(ETH_P_IP),
David S. Miller14e50e52007-05-24 18:17:54 -07002612 .destroy = ipv4_dst_destroy,
Jianzhao Wangae2688d2010-09-08 14:35:43 -07002613 .check = ipv4_blackhole_dst_check,
Steffen Klassertebb762f2011-11-23 02:12:51 +00002614 .mtu = ipv4_blackhole_mtu,
Eric Dumazet214f45c2011-02-18 11:39:01 -08002615 .default_advmss = ipv4_default_advmss,
David S. Miller14e50e52007-05-24 18:17:54 -07002616 .update_pmtu = ipv4_rt_blackhole_update_pmtu,
David S. Millerb587ee32012-07-12 00:39:24 -07002617 .redirect = ipv4_rt_blackhole_redirect,
Held Bernhard0972ddb2011-04-24 22:07:32 +00002618 .cow_metrics = ipv4_rt_blackhole_cow_metrics,
David S. Millerd3aaeb32011-07-18 00:40:17 -07002619 .neigh_lookup = ipv4_neigh_lookup,
David S. Miller14e50e52007-05-24 18:17:54 -07002620};
2621
David S. Miller2774c132011-03-01 14:59:04 -08002622struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
David S. Miller14e50e52007-05-24 18:17:54 -07002623{
David S. Miller5c1e6aa2011-04-28 14:13:38 -07002624 struct rtable *rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, 0, 0);
David S. Miller2774c132011-03-01 14:59:04 -08002625 struct rtable *ort = (struct rtable *) dst_orig;
David S. Miller14e50e52007-05-24 18:17:54 -07002626
2627 if (rt) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002628 struct dst_entry *new = &rt->dst;
David S. Miller14e50e52007-05-24 18:17:54 -07002629
David S. Miller14e50e52007-05-24 18:17:54 -07002630 new->__use = 1;
Herbert Xu352e5122007-11-13 21:34:06 -08002631 new->input = dst_discard;
2632 new->output = dst_discard;
David S. Miller14e50e52007-05-24 18:17:54 -07002633
Changli Gaod8d1f302010-06-10 23:31:35 -07002634 new->dev = ort->dst.dev;
David S. Miller14e50e52007-05-24 18:17:54 -07002635 if (new->dev)
2636 dev_hold(new->dev);
2637
David S. Miller5e2b61f2011-03-04 21:47:09 -08002638 rt->rt_key_dst = ort->rt_key_dst;
2639 rt->rt_key_src = ort->rt_key_src;
David S. Miller475949d2011-05-03 19:45:15 -07002640 rt->rt_key_tos = ort->rt_key_tos;
OGAWA Hirofumi1b86a582011-04-07 14:04:08 -07002641 rt->rt_route_iif = ort->rt_route_iif;
David S. Miller5e2b61f2011-03-04 21:47:09 -08002642 rt->rt_iif = ort->rt_iif;
2643 rt->rt_oif = ort->rt_oif;
2644 rt->rt_mark = ort->rt_mark;
David S. Miller59436342012-07-10 06:58:42 -07002645 rt->rt_pmtu = ort->rt_pmtu;
David S. Miller14e50e52007-05-24 18:17:54 -07002646
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002647 rt->rt_genid = rt_genid(net);
David S. Miller14e50e52007-05-24 18:17:54 -07002648 rt->rt_flags = ort->rt_flags;
2649 rt->rt_type = ort->rt_type;
2650 rt->rt_dst = ort->rt_dst;
2651 rt->rt_src = ort->rt_src;
David S. Miller14e50e52007-05-24 18:17:54 -07002652 rt->rt_gateway = ort->rt_gateway;
David S. Miller62fa8a82011-01-26 20:51:05 -08002653 rt->fi = ort->fi;
2654 if (rt->fi)
2655 atomic_inc(&rt->fi->fib_clntref);
David S. Miller14e50e52007-05-24 18:17:54 -07002656
2657 dst_free(new);
2658 }
2659
David S. Miller2774c132011-03-01 14:59:04 -08002660 dst_release(dst_orig);
2661
2662 return rt ? &rt->dst : ERR_PTR(-ENOMEM);
David S. Miller14e50e52007-05-24 18:17:54 -07002663}
2664
David S. Miller9d6ec932011-03-12 01:12:47 -05002665struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
David S. Millerb23dd4f2011-03-02 14:31:35 -08002666 struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002667{
David S. Miller9d6ec932011-03-12 01:12:47 -05002668 struct rtable *rt = __ip_route_output_key(net, flp4);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002669
David S. Millerb23dd4f2011-03-02 14:31:35 -08002670 if (IS_ERR(rt))
2671 return rt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002672
David S. Miller56157872011-05-02 14:37:45 -07002673 if (flp4->flowi4_proto)
David S. Miller9d6ec932011-03-12 01:12:47 -05002674 rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
2675 flowi4_to_flowi(flp4),
2676 sk, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002677
David S. Millerb23dd4f2011-03-02 14:31:35 -08002678 return rt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002679}
Arnaldo Carvalho de Melod8c97a92005-08-09 20:12:12 -07002680EXPORT_SYMBOL_GPL(ip_route_output_flow);
2681
Benjamin Thery4feb88e2009-01-22 04:56:23 +00002682static int rt_fill_info(struct net *net,
2683 struct sk_buff *skb, u32 pid, u32 seq, int event,
Jamal Hadi Salimb6544c02005-06-18 22:54:12 -07002684 int nowait, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002685{
Eric Dumazet511c3f92009-06-02 05:14:27 +00002686 struct rtable *rt = skb_rtable(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002687 struct rtmsg *r;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002688 struct nlmsghdr *nlh;
Steffen Klassert2bc8ca42011-10-11 01:12:02 +00002689 unsigned long expires = 0;
David S. Millerf1850712012-07-10 07:26:01 -07002690 u32 error;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002691
2692 nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2693 if (nlh == NULL)
Patrick McHardy26932562007-01-31 23:16:40 -08002694 return -EMSGSIZE;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002695
2696 r = nlmsg_data(nlh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002697 r->rtm_family = AF_INET;
2698 r->rtm_dst_len = 32;
2699 r->rtm_src_len = 0;
David S. Miller475949d2011-05-03 19:45:15 -07002700 r->rtm_tos = rt->rt_key_tos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002701 r->rtm_table = RT_TABLE_MAIN;
David S. Millerf3756b72012-04-01 20:39:02 -04002702 if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN))
2703 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002704 r->rtm_type = rt->rt_type;
2705 r->rtm_scope = RT_SCOPE_UNIVERSE;
2706 r->rtm_protocol = RTPROT_UNSPEC;
2707 r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2708 if (rt->rt_flags & RTCF_NOTIFY)
2709 r->rtm_flags |= RTM_F_NOTIFY;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002710
David S. Millerf3756b72012-04-01 20:39:02 -04002711 if (nla_put_be32(skb, RTA_DST, rt->rt_dst))
2712 goto nla_put_failure;
David S. Miller5e2b61f2011-03-04 21:47:09 -08002713 if (rt->rt_key_src) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002714 r->rtm_src_len = 32;
David S. Millerf3756b72012-04-01 20:39:02 -04002715 if (nla_put_be32(skb, RTA_SRC, rt->rt_key_src))
2716 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002717 }
David S. Millerf3756b72012-04-01 20:39:02 -04002718 if (rt->dst.dev &&
2719 nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2720 goto nla_put_failure;
Patrick McHardyc7066f72011-01-14 13:36:42 +01002721#ifdef CONFIG_IP_ROUTE_CLASSID
David S. Millerf3756b72012-04-01 20:39:02 -04002722 if (rt->dst.tclassid &&
2723 nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2724 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002725#endif
David S. Miller41347dc2012-06-28 04:05:27 -07002726 if (!rt_is_input_route(rt) &&
2727 rt->rt_src != rt->rt_key_src) {
David S. Millerf3756b72012-04-01 20:39:02 -04002728 if (nla_put_be32(skb, RTA_PREFSRC, rt->rt_src))
2729 goto nla_put_failure;
2730 }
2731 if (rt->rt_dst != rt->rt_gateway &&
2732 nla_put_be32(skb, RTA_GATEWAY, rt->rt_gateway))
2733 goto nla_put_failure;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002734
David S. Millerdefb3512010-12-08 21:16:57 -08002735 if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
Thomas Grafbe403ea2006-08-17 18:15:17 -07002736 goto nla_put_failure;
2737
David S. Millerf3756b72012-04-01 20:39:02 -04002738 if (rt->rt_mark &&
2739 nla_put_be32(skb, RTA_MARK, rt->rt_mark))
2740 goto nla_put_failure;
Eric Dumazet963bfee2010-07-20 22:03:14 +00002741
Changli Gaod8d1f302010-06-10 23:31:35 -07002742 error = rt->dst.error;
David S. Miller59436342012-07-10 06:58:42 -07002743 expires = rt->dst.expires;
2744 if (expires) {
2745 if (time_before(jiffies, expires))
2746 expires -= jiffies;
2747 else
2748 expires = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002749 }
Thomas Grafbe403ea2006-08-17 18:15:17 -07002750
David S. Millerc7537962010-11-11 17:07:48 -08002751 if (rt_is_input_route(rt)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002752#ifdef CONFIG_IP_MROUTE
Al Viroe4485152006-09-26 22:15:01 -07002753 __be32 dst = rt->rt_dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002754
Joe Perchesf97c1e02007-12-16 13:45:43 -08002755 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
Benjamin Thery4feb88e2009-01-22 04:56:23 +00002756 IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
David S. Miller9a1b9492011-05-04 12:18:54 -07002757 int err = ipmr_get_route(net, skb,
2758 rt->rt_src, rt->rt_dst,
2759 r, nowait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002760 if (err <= 0) {
2761 if (!nowait) {
2762 if (err == 0)
2763 return 0;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002764 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002765 } else {
2766 if (err == -EMSGSIZE)
Thomas Grafbe403ea2006-08-17 18:15:17 -07002767 goto nla_put_failure;
Thomas Grafe3703b32006-11-27 09:27:07 -08002768 error = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002769 }
2770 }
2771 } else
2772#endif
David S. Millerf3756b72012-04-01 20:39:02 -04002773 if (nla_put_u32(skb, RTA_IIF, rt->rt_iif))
2774 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002775 }
2776
David S. Millerf1850712012-07-10 07:26:01 -07002777 if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
Thomas Grafe3703b32006-11-27 09:27:07 -08002778 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002779
Thomas Grafbe403ea2006-08-17 18:15:17 -07002780 return nlmsg_end(skb, nlh);
2781
2782nla_put_failure:
Patrick McHardy26932562007-01-31 23:16:40 -08002783 nlmsg_cancel(skb, nlh);
2784 return -EMSGSIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002785}
2786
Daniel Baluta5e73ea12012-04-15 01:34:41 +00002787static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, void *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002788{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002789 struct net *net = sock_net(in_skb->sk);
Thomas Grafd889ce32006-08-17 18:15:44 -07002790 struct rtmsg *rtm;
2791 struct nlattr *tb[RTA_MAX+1];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002792 struct rtable *rt = NULL;
Al Viro9e12bb22006-09-26 21:25:20 -07002793 __be32 dst = 0;
2794 __be32 src = 0;
2795 u32 iif;
Thomas Grafd889ce32006-08-17 18:15:44 -07002796 int err;
Eric Dumazet963bfee2010-07-20 22:03:14 +00002797 int mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002798 struct sk_buff *skb;
2799
Thomas Grafd889ce32006-08-17 18:15:44 -07002800 err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2801 if (err < 0)
2802 goto errout;
2803
2804 rtm = nlmsg_data(nlh);
2805
Linus Torvalds1da177e2005-04-16 15:20:36 -07002806 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
Thomas Grafd889ce32006-08-17 18:15:44 -07002807 if (skb == NULL) {
2808 err = -ENOBUFS;
2809 goto errout;
2810 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002811
2812 /* Reserve room for dummy headers, this skb can pass
2813 through good chunk of routing engine.
2814 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07002815 skb_reset_mac_header(skb);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07002816 skb_reset_network_header(skb);
Stephen Hemmingerd2c962b2006-04-17 17:27:11 -07002817
2818 /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07002819 ip_hdr(skb)->protocol = IPPROTO_ICMP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002820 skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2821
Al Viro17fb2c62006-09-26 22:15:25 -07002822 src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2823 dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
Thomas Grafd889ce32006-08-17 18:15:44 -07002824 iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
Eric Dumazet963bfee2010-07-20 22:03:14 +00002825 mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002826
2827 if (iif) {
Thomas Grafd889ce32006-08-17 18:15:44 -07002828 struct net_device *dev;
2829
Denis V. Lunev19375042008-02-28 20:52:04 -08002830 dev = __dev_get_by_index(net, iif);
Thomas Grafd889ce32006-08-17 18:15:44 -07002831 if (dev == NULL) {
2832 err = -ENODEV;
2833 goto errout_free;
2834 }
2835
Linus Torvalds1da177e2005-04-16 15:20:36 -07002836 skb->protocol = htons(ETH_P_IP);
2837 skb->dev = dev;
Eric Dumazet963bfee2010-07-20 22:03:14 +00002838 skb->mark = mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002839 local_bh_disable();
2840 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2841 local_bh_enable();
Thomas Grafd889ce32006-08-17 18:15:44 -07002842
Eric Dumazet511c3f92009-06-02 05:14:27 +00002843 rt = skb_rtable(skb);
Changli Gaod8d1f302010-06-10 23:31:35 -07002844 if (err == 0 && rt->dst.error)
2845 err = -rt->dst.error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002846 } else {
David S. Miller68a5e3d2011-03-11 20:07:33 -05002847 struct flowi4 fl4 = {
2848 .daddr = dst,
2849 .saddr = src,
2850 .flowi4_tos = rtm->rtm_tos,
2851 .flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
2852 .flowi4_mark = mark,
Thomas Grafd889ce32006-08-17 18:15:44 -07002853 };
David S. Miller9d6ec932011-03-12 01:12:47 -05002854 rt = ip_route_output_key(net, &fl4);
David S. Millerb23dd4f2011-03-02 14:31:35 -08002855
2856 err = 0;
2857 if (IS_ERR(rt))
2858 err = PTR_ERR(rt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002859 }
Thomas Grafd889ce32006-08-17 18:15:44 -07002860
Linus Torvalds1da177e2005-04-16 15:20:36 -07002861 if (err)
Thomas Grafd889ce32006-08-17 18:15:44 -07002862 goto errout_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002863
Changli Gaod8d1f302010-06-10 23:31:35 -07002864 skb_dst_set(skb, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002865 if (rtm->rtm_flags & RTM_F_NOTIFY)
2866 rt->rt_flags |= RTCF_NOTIFY;
2867
Benjamin Thery4feb88e2009-01-22 04:56:23 +00002868 err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
Denis V. Lunev19375042008-02-28 20:52:04 -08002869 RTM_NEWROUTE, 0, 0);
Thomas Grafd889ce32006-08-17 18:15:44 -07002870 if (err <= 0)
2871 goto errout_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002872
Denis V. Lunev19375042008-02-28 20:52:04 -08002873 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
Thomas Grafd889ce32006-08-17 18:15:44 -07002874errout:
Thomas Graf2942e902006-08-15 00:30:25 -07002875 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002876
Thomas Grafd889ce32006-08-17 18:15:44 -07002877errout_free:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002878 kfree_skb(skb);
Thomas Grafd889ce32006-08-17 18:15:44 -07002879 goto errout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002880}
2881
2882int ip_rt_dump(struct sk_buff *skb, struct netlink_callback *cb)
2883{
2884 struct rtable *rt;
2885 int h, s_h;
2886 int idx, s_idx;
Denis V. Lunev19375042008-02-28 20:52:04 -08002887 struct net *net;
2888
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002889 net = sock_net(skb->sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002890
2891 s_h = cb->args[0];
Eric Dumazetd8c92832008-01-07 21:52:14 -08002892 if (s_h < 0)
2893 s_h = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002894 s_idx = idx = cb->args[1];
Eric Dumazeta6272662008-08-28 01:11:25 -07002895 for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
2896 if (!rt_hash_table[h].chain)
2897 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002898 rcu_read_lock_bh();
Paul E. McKenneya898def2010-02-22 17:04:49 -08002899 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
Changli Gaod8d1f302010-06-10 23:31:35 -07002900 rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
2901 if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002902 continue;
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002903 if (rt_is_expired(rt))
Eric Dumazet29e75252008-01-31 17:05:09 -08002904 continue;
Changli Gaod8d1f302010-06-10 23:31:35 -07002905 skb_dst_set_noref(skb, &rt->dst);
Benjamin Thery4feb88e2009-01-22 04:56:23 +00002906 if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002907 cb->nlh->nlmsg_seq, RTM_NEWROUTE,
Jamal Hadi Salimb6544c02005-06-18 22:54:12 -07002908 1, NLM_F_MULTI) <= 0) {
Eric Dumazetadf30902009-06-02 05:19:30 +00002909 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002910 rcu_read_unlock_bh();
2911 goto done;
2912 }
Eric Dumazetadf30902009-06-02 05:19:30 +00002913 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002914 }
2915 rcu_read_unlock_bh();
2916 }
2917
2918done:
2919 cb->args[0] = h;
2920 cb->args[1] = idx;
2921 return skb->len;
2922}
2923
2924void ip_rt_multicast_event(struct in_device *in_dev)
2925{
Denis V. Lunev76e6ebf2008-07-05 19:00:44 -07002926 rt_cache_flush(dev_net(in_dev->dev), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002927}
2928
2929#ifdef CONFIG_SYSCTL
Denis V. Lunev81c684d2008-07-08 03:05:28 -07002930static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07002931 void __user *buffer,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002932 size_t *lenp, loff_t *ppos)
2933{
2934 if (write) {
Denis V. Lunev639e1042008-07-05 19:02:06 -07002935 int flush_delay;
Denis V. Lunev81c684d2008-07-08 03:05:28 -07002936 ctl_table ctl;
Denis V. Lunev39a23e72008-07-05 19:02:33 -07002937 struct net *net;
Denis V. Lunev639e1042008-07-05 19:02:06 -07002938
Denis V. Lunev81c684d2008-07-08 03:05:28 -07002939 memcpy(&ctl, __ctl, sizeof(ctl));
2940 ctl.data = &flush_delay;
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07002941 proc_dointvec(&ctl, write, buffer, lenp, ppos);
Denis V. Lunev639e1042008-07-05 19:02:06 -07002942
Denis V. Lunev81c684d2008-07-08 03:05:28 -07002943 net = (struct net *)__ctl->extra1;
Denis V. Lunev39a23e72008-07-05 19:02:33 -07002944 rt_cache_flush(net, flush_delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002945 return 0;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002946 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002947
2948 return -EINVAL;
2949}
2950
Al Viroeeb61f72008-07-27 08:59:33 +01002951static ctl_table ipv4_route_table[] = {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002952 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002953 .procname = "gc_thresh",
2954 .data = &ipv4_dst_ops.gc_thresh,
2955 .maxlen = sizeof(int),
2956 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08002957 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002958 },
2959 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002960 .procname = "max_size",
2961 .data = &ip_rt_max_size,
2962 .maxlen = sizeof(int),
2963 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08002964 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002965 },
2966 {
2967 /* Deprecated. Use gc_min_interval_ms */
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002968
Linus Torvalds1da177e2005-04-16 15:20:36 -07002969 .procname = "gc_min_interval",
2970 .data = &ip_rt_gc_min_interval,
2971 .maxlen = sizeof(int),
2972 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08002973 .proc_handler = proc_dointvec_jiffies,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002974 },
2975 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002976 .procname = "gc_min_interval_ms",
2977 .data = &ip_rt_gc_min_interval,
2978 .maxlen = sizeof(int),
2979 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08002980 .proc_handler = proc_dointvec_ms_jiffies,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002981 },
2982 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002983 .procname = "gc_timeout",
2984 .data = &ip_rt_gc_timeout,
2985 .maxlen = sizeof(int),
2986 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08002987 .proc_handler = proc_dointvec_jiffies,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002988 },
2989 {
Eric Dumazet9f28a2f2011-12-21 15:47:16 -05002990 .procname = "gc_interval",
2991 .data = &ip_rt_gc_interval,
2992 .maxlen = sizeof(int),
2993 .mode = 0644,
2994 .proc_handler = proc_dointvec_jiffies,
2995 },
2996 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002997 .procname = "redirect_load",
2998 .data = &ip_rt_redirect_load,
2999 .maxlen = sizeof(int),
3000 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003001 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003002 },
3003 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003004 .procname = "redirect_number",
3005 .data = &ip_rt_redirect_number,
3006 .maxlen = sizeof(int),
3007 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003008 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003009 },
3010 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003011 .procname = "redirect_silence",
3012 .data = &ip_rt_redirect_silence,
3013 .maxlen = sizeof(int),
3014 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003015 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003016 },
3017 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003018 .procname = "error_cost",
3019 .data = &ip_rt_error_cost,
3020 .maxlen = sizeof(int),
3021 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003022 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003023 },
3024 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003025 .procname = "error_burst",
3026 .data = &ip_rt_error_burst,
3027 .maxlen = sizeof(int),
3028 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003029 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003030 },
3031 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003032 .procname = "gc_elasticity",
3033 .data = &ip_rt_gc_elasticity,
3034 .maxlen = sizeof(int),
3035 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003036 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003037 },
3038 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003039 .procname = "mtu_expires",
3040 .data = &ip_rt_mtu_expires,
3041 .maxlen = sizeof(int),
3042 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003043 .proc_handler = proc_dointvec_jiffies,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003044 },
3045 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003046 .procname = "min_pmtu",
3047 .data = &ip_rt_min_pmtu,
3048 .maxlen = sizeof(int),
3049 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003050 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003051 },
3052 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003053 .procname = "min_adv_mss",
3054 .data = &ip_rt_min_advmss,
3055 .maxlen = sizeof(int),
3056 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003057 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058 },
Eric W. Biedermanf8572d82009-11-05 13:32:03 -08003059 { }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003060};
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003061
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003062static struct ctl_table ipv4_route_flush_table[] = {
3063 {
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003064 .procname = "flush",
3065 .maxlen = sizeof(int),
3066 .mode = 0200,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003067 .proc_handler = ipv4_sysctl_rtcache_flush,
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003068 },
Eric W. Biedermanf8572d82009-11-05 13:32:03 -08003069 { },
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003070};
3071
3072static __net_init int sysctl_route_net_init(struct net *net)
3073{
3074 struct ctl_table *tbl;
3075
3076 tbl = ipv4_route_flush_table;
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08003077 if (!net_eq(net, &init_net)) {
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003078 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3079 if (tbl == NULL)
3080 goto err_dup;
3081 }
3082 tbl[0].extra1 = net;
3083
Eric W. Biedermanec8f23c2012-04-19 13:44:49 +00003084 net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003085 if (net->ipv4.route_hdr == NULL)
3086 goto err_reg;
3087 return 0;
3088
3089err_reg:
3090 if (tbl != ipv4_route_flush_table)
3091 kfree(tbl);
3092err_dup:
3093 return -ENOMEM;
3094}
3095
3096static __net_exit void sysctl_route_net_exit(struct net *net)
3097{
3098 struct ctl_table *tbl;
3099
3100 tbl = net->ipv4.route_hdr->ctl_table_arg;
3101 unregister_net_sysctl_table(net->ipv4.route_hdr);
3102 BUG_ON(tbl == ipv4_route_flush_table);
3103 kfree(tbl);
3104}
3105
3106static __net_initdata struct pernet_operations sysctl_route_ops = {
3107 .init = sysctl_route_net_init,
3108 .exit = sysctl_route_net_exit,
3109};
Linus Torvalds1da177e2005-04-16 15:20:36 -07003110#endif
3111
Neil Horman3ee94372010-05-08 01:57:52 -07003112static __net_init int rt_genid_init(struct net *net)
Denis V. Lunev9f5e97e2008-07-05 19:02:59 -07003113{
Neil Horman3ee94372010-05-08 01:57:52 -07003114 get_random_bytes(&net->ipv4.rt_genid,
3115 sizeof(net->ipv4.rt_genid));
David S. Miller436c3b62011-03-24 17:42:21 -07003116 get_random_bytes(&net->ipv4.dev_addr_genid,
3117 sizeof(net->ipv4.dev_addr_genid));
Denis V. Lunev9f5e97e2008-07-05 19:02:59 -07003118 return 0;
3119}
3120
Neil Horman3ee94372010-05-08 01:57:52 -07003121static __net_initdata struct pernet_operations rt_genid_ops = {
3122 .init = rt_genid_init,
Denis V. Lunev9f5e97e2008-07-05 19:02:59 -07003123};
3124
David S. Millerc3426b42012-06-09 16:27:05 -07003125static int __net_init ipv4_inetpeer_init(struct net *net)
3126{
3127 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3128
3129 if (!bp)
3130 return -ENOMEM;
3131 inet_peer_base_init(bp);
3132 net->ipv4.peers = bp;
3133 return 0;
3134}
3135
3136static void __net_exit ipv4_inetpeer_exit(struct net *net)
3137{
3138 struct inet_peer_base *bp = net->ipv4.peers;
3139
3140 net->ipv4.peers = NULL;
David S. Miller56a6b242012-06-09 16:32:41 -07003141 inetpeer_invalidate_tree(bp);
David S. Millerc3426b42012-06-09 16:27:05 -07003142 kfree(bp);
3143}
3144
3145static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3146 .init = ipv4_inetpeer_init,
3147 .exit = ipv4_inetpeer_exit,
3148};
Denis V. Lunev9f5e97e2008-07-05 19:02:59 -07003149
Patrick McHardyc7066f72011-01-14 13:36:42 +01003150#ifdef CONFIG_IP_ROUTE_CLASSID
Tejun Heo7d720c32010-02-16 15:20:26 +00003151struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
Patrick McHardyc7066f72011-01-14 13:36:42 +01003152#endif /* CONFIG_IP_ROUTE_CLASSID */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003153
3154static __initdata unsigned long rhash_entries;
3155static int __init set_rhash_entries(char *str)
3156{
Eldad Zack413c27d2012-05-19 14:13:18 +00003157 ssize_t ret;
3158
Linus Torvalds1da177e2005-04-16 15:20:36 -07003159 if (!str)
3160 return 0;
Eldad Zack413c27d2012-05-19 14:13:18 +00003161
3162 ret = kstrtoul(str, 0, &rhash_entries);
3163 if (ret)
3164 return 0;
3165
Linus Torvalds1da177e2005-04-16 15:20:36 -07003166 return 1;
3167}
3168__setup("rhash_entries=", set_rhash_entries);
3169
3170int __init ip_rt_init(void)
3171{
Eric Dumazet424c4b72005-07-05 14:58:19 -07003172 int rc = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003173
Patrick McHardyc7066f72011-01-14 13:36:42 +01003174#ifdef CONFIG_IP_ROUTE_CLASSID
Ingo Molnar0dcec8c2009-02-25 14:07:33 +01003175 ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003176 if (!ip_rt_acct)
3177 panic("IP: failed to allocate ip_rt_acct\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003178#endif
3179
Alexey Dobriyane5d679f332006-08-26 19:25:52 -07003180 ipv4_dst_ops.kmem_cachep =
3181 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
Paul Mundt20c2df82007-07-20 10:11:58 +09003182 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003183
David S. Miller14e50e52007-05-24 18:17:54 -07003184 ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3185
Eric Dumazetfc66f952010-10-08 06:37:34 +00003186 if (dst_entries_init(&ipv4_dst_ops) < 0)
3187 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3188
3189 if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3190 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3191
Eric Dumazet424c4b72005-07-05 14:58:19 -07003192 rt_hash_table = (struct rt_hash_bucket *)
3193 alloc_large_system_hash("IP route cache",
3194 sizeof(struct rt_hash_bucket),
3195 rhash_entries,
Jan Beulich44813742009-09-21 17:03:05 -07003196 (totalram_pages >= 128 * 1024) ?
Mike Stroyan18955cf2005-11-29 16:12:55 -08003197 15 : 17,
Kirill Korotaev8d1502d2006-08-07 20:44:22 -07003198 0,
Eric Dumazet424c4b72005-07-05 14:58:19 -07003199 &rt_hash_log,
3200 &rt_hash_mask,
Tim Bird31fe62b2012-05-23 13:33:35 +00003201 0,
Anton Blanchardc9503e02009-04-27 05:42:24 -07003202 rhash_entries ? 0 : 512 * 1024);
Eric Dumazet22c047c2005-07-05 14:55:24 -07003203 memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
3204 rt_hash_lock_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003205
3206 ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
3207 ip_rt_max_size = (rt_hash_mask + 1) * 16;
3208
Linus Torvalds1da177e2005-04-16 15:20:36 -07003209 devinet_init();
3210 ip_fib_init();
3211
Eric Dumazet9f28a2f2011-12-21 15:47:16 -05003212 INIT_DELAYED_WORK_DEFERRABLE(&expires_work, rt_worker_func);
3213 expires_ljiffies = jiffies;
3214 schedule_delayed_work(&expires_work,
3215 net_random() % ip_rt_gc_interval + ip_rt_gc_interval);
3216
Denis V. Lunev73b38712008-02-28 20:51:18 -08003217 if (ip_rt_proc_init())
Joe Perches058bd4d2012-03-11 18:36:11 +00003218 pr_err("Unable to create route proc files\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003219#ifdef CONFIG_XFRM
3220 xfrm_init();
Neil Hormana33bc5c2009-07-30 18:52:15 -07003221 xfrm4_init(ip_rt_max_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003222#endif
Greg Rosec7ac8672011-06-10 01:27:09 +00003223 rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
Thomas Graf63f34442007-03-22 11:55:17 -07003224
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003225#ifdef CONFIG_SYSCTL
3226 register_pernet_subsys(&sysctl_route_ops);
3227#endif
Neil Horman3ee94372010-05-08 01:57:52 -07003228 register_pernet_subsys(&rt_genid_ops);
David S. Millerc3426b42012-06-09 16:27:05 -07003229 register_pernet_subsys(&ipv4_inetpeer_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003230 return rc;
3231}
3232
Al Viroa1bc6eb2008-07-30 06:32:52 -04003233#ifdef CONFIG_SYSCTL
Al Viroeeb61f72008-07-27 08:59:33 +01003234/*
3235 * We really need to sanitize the damn ipv4 init order, then all
3236 * this nonsense will go away.
3237 */
3238void __init ip_static_sysctl_init(void)
3239{
Eric W. Biederman4e5ca782012-04-19 13:32:39 +00003240 register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
Al Viroeeb61f72008-07-27 08:59:33 +01003241}
Al Viroa1bc6eb2008-07-30 06:32:52 -04003242#endif