aboutsummaryrefslogtreecommitdiff
path: root/net/ipv4/tcp_metrics.c
diff options
context:
space:
mode:
Diffstat (limited to 'net/ipv4/tcp_metrics.c')
-rw-r--r--net/ipv4/tcp_metrics.c208
1 files changed, 108 insertions, 100 deletions
diff --git a/net/ipv4/tcp_metrics.c b/net/ipv4/tcp_metrics.c
index e5f41bd5ec1b..a51d63a43e33 100644
--- a/net/ipv4/tcp_metrics.c
+++ b/net/ipv4/tcp_metrics.c
@@ -28,7 +28,8 @@ static struct tcp_metrics_block *__tcp_get_metrics(const struct inetpeer_addr *s
struct tcp_fastopen_metrics {
u16 mss;
- u16 syn_loss:10; /* Recurring Fast Open SYN losses */
+ u16 syn_loss:10, /* Recurring Fast Open SYN losses */
+ try_exp:2; /* Request w/ exp. option (once) */
unsigned long last_syn_loss; /* Last Fast Open SYN loss */
struct tcp_fastopen_cookie cookie;
};
@@ -40,6 +41,7 @@ struct tcp_fastopen_metrics {
struct tcp_metrics_block {
struct tcp_metrics_block __rcu *tcpm_next;
+ possible_net_t tcpm_net;
struct inetpeer_addr tcpm_saddr;
struct inetpeer_addr tcpm_daddr;
unsigned long tcpm_stamp;
@@ -52,6 +54,11 @@ struct tcp_metrics_block {
struct rcu_head rcu_head;
};
+static inline struct net *tm_net(struct tcp_metrics_block *tm)
+{
+ return read_pnet(&tm->tcpm_net);
+}
+
static bool tcp_metric_locked(struct tcp_metrics_block *tm,
enum tcp_metric_index idx)
{
@@ -74,23 +81,20 @@ static void tcp_metric_set(struct tcp_metrics_block *tm,
static bool addr_same(const struct inetpeer_addr *a,
const struct inetpeer_addr *b)
{
- const struct in6_addr *a6, *b6;
-
if (a->family != b->family)
return false;
if (a->family == AF_INET)
return a->addr.a4 == b->addr.a4;
-
- a6 = (const struct in6_addr *) &a->addr.a6[0];
- b6 = (const struct in6_addr *) &b->addr.a6[0];
-
- return ipv6_addr_equal(a6, b6);
+ return ipv6_addr_equal(&a->addr.in6, &b->addr.in6);
}
struct tcpm_hash_bucket {
struct tcp_metrics_block __rcu *chain;
};
+static struct tcpm_hash_bucket *tcp_metrics_hash __read_mostly;
+static unsigned int tcp_metrics_hash_log __read_mostly;
+
static DEFINE_SPINLOCK(tcp_metrics_lock);
static void tcpm_suck_dst(struct tcp_metrics_block *tm,
@@ -128,6 +132,8 @@ static void tcpm_suck_dst(struct tcp_metrics_block *tm,
if (fastopen_clear) {
tm->tcpm_fastopen.mss = 0;
tm->tcpm_fastopen.syn_loss = 0;
+ tm->tcpm_fastopen.try_exp = 0;
+ tm->tcpm_fastopen.cookie.exp = false;
tm->tcpm_fastopen.cookie.len = 0;
}
}
@@ -143,6 +149,9 @@ static void tcpm_check_stamp(struct tcp_metrics_block *tm, struct dst_entry *dst
#define TCP_METRICS_RECLAIM_DEPTH 5
#define TCP_METRICS_RECLAIM_PTR (struct tcp_metrics_block *) 0x1UL
+#define deref_locked(p) \
+ rcu_dereference_protected(p, lockdep_is_held(&tcp_metrics_lock))
+
static struct tcp_metrics_block *tcpm_new(struct dst_entry *dst,
struct inetpeer_addr *saddr,
struct inetpeer_addr *daddr,
@@ -171,9 +180,9 @@ static struct tcp_metrics_block *tcpm_new(struct dst_entry *dst,
if (unlikely(reclaim)) {
struct tcp_metrics_block *oldest;
- oldest = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain);
- for (tm = rcu_dereference(oldest->tcpm_next); tm;
- tm = rcu_dereference(tm->tcpm_next)) {
+ oldest = deref_locked(tcp_metrics_hash[hash].chain);
+ for (tm = deref_locked(oldest->tcpm_next); tm;
+ tm = deref_locked(tm->tcpm_next)) {
if (time_before(tm->tcpm_stamp, oldest->tcpm_stamp))
oldest = tm;
}
@@ -183,14 +192,15 @@ static struct tcp_metrics_block *tcpm_new(struct dst_entry *dst,
if (!tm)
goto out_unlock;
}
+ write_pnet(&tm->tcpm_net, net);
tm->tcpm_saddr = *saddr;
tm->tcpm_daddr = *daddr;
tcpm_suck_dst(tm, dst, true);
if (likely(!reclaim)) {
- tm->tcpm_next = net->ipv4.tcp_metrics_hash[hash].chain;
- rcu_assign_pointer(net->ipv4.tcp_metrics_hash[hash].chain, tm);
+ tm->tcpm_next = tcp_metrics_hash[hash].chain;
+ rcu_assign_pointer(tcp_metrics_hash[hash].chain, tm);
}
out_unlock:
@@ -214,10 +224,11 @@ static struct tcp_metrics_block *__tcp_get_metrics(const struct inetpeer_addr *s
struct tcp_metrics_block *tm;
int depth = 0;
- for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
+ for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;
tm = rcu_dereference(tm->tcpm_next)) {
if (addr_same(&tm->tcpm_saddr, saddr) &&
- addr_same(&tm->tcpm_daddr, daddr))
+ addr_same(&tm->tcpm_daddr, daddr) &&
+ net_eq(tm_net(tm), net))
break;
depth++;
}
@@ -242,8 +253,8 @@ static struct tcp_metrics_block *__tcp_get_metrics_req(struct request_sock *req,
break;
#if IS_ENABLED(CONFIG_IPV6)
case AF_INET6:
- *(struct in6_addr *)saddr.addr.a6 = inet_rsk(req)->ir_v6_loc_addr;
- *(struct in6_addr *)daddr.addr.a6 = inet_rsk(req)->ir_v6_rmt_addr;
+ saddr.addr.in6 = inet_rsk(req)->ir_v6_loc_addr;
+ daddr.addr.in6 = inet_rsk(req)->ir_v6_rmt_addr;
hash = ipv6_addr_hash(&inet_rsk(req)->ir_v6_rmt_addr);
break;
#endif
@@ -252,12 +263,14 @@ static struct tcp_metrics_block *__tcp_get_metrics_req(struct request_sock *req,
}
net = dev_net(dst->dev);
- hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
+ hash ^= net_hash_mix(net);
+ hash = hash_32(hash, tcp_metrics_hash_log);
- for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
+ for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;
tm = rcu_dereference(tm->tcpm_next)) {
if (addr_same(&tm->tcpm_saddr, &saddr) &&
- addr_same(&tm->tcpm_daddr, &daddr))
+ addr_same(&tm->tcpm_daddr, &daddr) &&
+ net_eq(tm_net(tm), net))
break;
}
tcpm_check_stamp(tm, dst);
@@ -288,9 +301,9 @@ static struct tcp_metrics_block *__tcp_get_metrics_tw(struct inet_timewait_sock
hash = (__force unsigned int) daddr.addr.a4;
} else {
saddr.family = AF_INET6;
- *(struct in6_addr *)saddr.addr.a6 = tw->tw_v6_rcv_saddr;
+ saddr.addr.in6 = tw->tw_v6_rcv_saddr;
daddr.family = AF_INET6;
- *(struct in6_addr *)daddr.addr.a6 = tw->tw_v6_daddr;
+ daddr.addr.in6 = tw->tw_v6_daddr;
hash = ipv6_addr_hash(&tw->tw_v6_daddr);
}
}
@@ -299,12 +312,14 @@ static struct tcp_metrics_block *__tcp_get_metrics_tw(struct inet_timewait_sock
return NULL;
net = twsk_net(tw);
- hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
+ hash ^= net_hash_mix(net);
+ hash = hash_32(hash, tcp_metrics_hash_log);
- for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
+ for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;
tm = rcu_dereference(tm->tcpm_next)) {
if (addr_same(&tm->tcpm_saddr, &saddr) &&
- addr_same(&tm->tcpm_daddr, &daddr))
+ addr_same(&tm->tcpm_daddr, &daddr) &&
+ net_eq(tm_net(tm), net))
break;
}
return tm;
@@ -336,9 +351,9 @@ static struct tcp_metrics_block *tcp_get_metrics(struct sock *sk,
hash = (__force unsigned int) daddr.addr.a4;
} else {
saddr.family = AF_INET6;
- *(struct in6_addr *)saddr.addr.a6 = sk->sk_v6_rcv_saddr;
+ saddr.addr.in6 = sk->sk_v6_rcv_saddr;
daddr.family = AF_INET6;
- *(struct in6_addr *)daddr.addr.a6 = sk->sk_v6_daddr;
+ daddr.addr.in6 = sk->sk_v6_daddr;
hash = ipv6_addr_hash(&sk->sk_v6_daddr);
}
}
@@ -347,7 +362,8 @@ static struct tcp_metrics_block *tcp_get_metrics(struct sock *sk,
return NULL;
net = dev_net(dst->dev);
- hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
+ hash ^= net_hash_mix(net);
+ hash = hash_32(hash, tcp_metrics_hash_log);
tm = __tcp_get_metrics(&saddr, &daddr, net, hash);
if (tm == TCP_METRICS_RECLAIM_PTR)
@@ -492,7 +508,7 @@ void tcp_init_metrics(struct sock *sk)
struct tcp_metrics_block *tm;
u32 val, crtt = 0; /* cached RTT scaled by 8 */
- if (dst == NULL)
+ if (!dst)
goto reset;
dst_confirm(dst);
@@ -700,6 +716,8 @@ void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
if (tfom->mss)
*mss = tfom->mss;
*cookie = tfom->cookie;
+ if (cookie->len <= 0 && tfom->try_exp == 1)
+ cookie->exp = true;
*syn_loss = tfom->syn_loss;
*last_syn_loss = *syn_loss ? tfom->last_syn_loss : 0;
} while (read_seqretry(&fastopen_seqlock, seq));
@@ -708,7 +726,8 @@ void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
}
void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
- struct tcp_fastopen_cookie *cookie, bool syn_lost)
+ struct tcp_fastopen_cookie *cookie, bool syn_lost,
+ u16 try_exp)
{
struct dst_entry *dst = __sk_dst_get(sk);
struct tcp_metrics_block *tm;
@@ -725,6 +744,9 @@ void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
tfom->mss = mss;
if (cookie && cookie->len > 0)
tfom->cookie = *cookie;
+ else if (try_exp > tfom->try_exp &&
+ tfom->cookie.len <= 0 && !tfom->cookie.exp)
+ tfom->try_exp = try_exp;
if (syn_lost) {
++tfom->syn_loss;
tfom->last_syn_loss = jiffies;
@@ -773,19 +795,19 @@ static int tcp_metrics_fill_info(struct sk_buff *msg,
switch (tm->tcpm_daddr.family) {
case AF_INET:
- if (nla_put_be32(msg, TCP_METRICS_ATTR_ADDR_IPV4,
- tm->tcpm_daddr.addr.a4) < 0)
+ if (nla_put_in_addr(msg, TCP_METRICS_ATTR_ADDR_IPV4,
+ tm->tcpm_daddr.addr.a4) < 0)
goto nla_put_failure;
- if (nla_put_be32(msg, TCP_METRICS_ATTR_SADDR_IPV4,
- tm->tcpm_saddr.addr.a4) < 0)
+ if (nla_put_in_addr(msg, TCP_METRICS_ATTR_SADDR_IPV4,
+ tm->tcpm_saddr.addr.a4) < 0)
goto nla_put_failure;
break;
case AF_INET6:
- if (nla_put(msg, TCP_METRICS_ATTR_ADDR_IPV6, 16,
- tm->tcpm_daddr.addr.a6) < 0)
+ if (nla_put_in6_addr(msg, TCP_METRICS_ATTR_ADDR_IPV6,
+ &tm->tcpm_daddr.addr.in6) < 0)
goto nla_put_failure;
- if (nla_put(msg, TCP_METRICS_ATTR_SADDR_IPV6, 16,
- tm->tcpm_saddr.addr.a6) < 0)
+ if (nla_put_in6_addr(msg, TCP_METRICS_ATTR_SADDR_IPV6,
+ &tm->tcpm_saddr.addr.in6) < 0)
goto nla_put_failure;
break;
default:
@@ -898,17 +920,19 @@ static int tcp_metrics_nl_dump(struct sk_buff *skb,
struct netlink_callback *cb)
{
struct net *net = sock_net(skb->sk);
- unsigned int max_rows = 1U << net->ipv4.tcp_metrics_hash_log;
+ unsigned int max_rows = 1U << tcp_metrics_hash_log;
unsigned int row, s_row = cb->args[0];
int s_col = cb->args[1], col = s_col;
for (row = s_row; row < max_rows; row++, s_col = 0) {
struct tcp_metrics_block *tm;
- struct tcpm_hash_bucket *hb = net->ipv4.tcp_metrics_hash + row;
+ struct tcpm_hash_bucket *hb = tcp_metrics_hash + row;
rcu_read_lock();
for (col = 0, tm = rcu_dereference(hb->chain); tm;
tm = rcu_dereference(tm->tcpm_next), col++) {
+ if (!net_eq(tm_net(tm), net))
+ continue;
if (col < s_col)
continue;
if (tcp_metrics_dump_info(skb, cb, tm) < 0) {
@@ -933,7 +957,7 @@ static int __parse_nl_addr(struct genl_info *info, struct inetpeer_addr *addr,
a = info->attrs[v4];
if (a) {
addr->family = AF_INET;
- addr->addr.a4 = nla_get_be32(a);
+ addr->addr.a4 = nla_get_in_addr(a);
if (hash)
*hash = (__force unsigned int) addr->addr.a4;
return 0;
@@ -943,9 +967,9 @@ static int __parse_nl_addr(struct genl_info *info, struct inetpeer_addr *addr,
if (nla_len(a) != sizeof(struct in6_addr))
return -EINVAL;
addr->family = AF_INET6;
- memcpy(addr->addr.a6, nla_data(a), sizeof(addr->addr.a6));
+ addr->addr.in6 = nla_get_in6_addr(a);
if (hash)
- *hash = ipv6_addr_hash((struct in6_addr *) addr->addr.a6);
+ *hash = ipv6_addr_hash(&addr->addr.in6);
return 0;
}
return optional ? 1 : -EAFNOSUPPORT;
@@ -994,13 +1018,15 @@ static int tcp_metrics_nl_cmd_get(struct sk_buff *skb, struct genl_info *info)
if (!reply)
goto nla_put_failure;
- hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
+ hash ^= net_hash_mix(net);
+ hash = hash_32(hash, tcp_metrics_hash_log);
ret = -ESRCH;
rcu_read_lock();
- for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
+ for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;
tm = rcu_dereference(tm->tcpm_next)) {
if (addr_same(&tm->tcpm_daddr, &daddr) &&
- (!src || addr_same(&tm->tcpm_saddr, &saddr))) {
+ (!src || addr_same(&tm->tcpm_saddr, &saddr)) &&
+ net_eq(tm_net(tm), net)) {
ret = tcp_metrics_fill_info(msg, tm);
break;
}
@@ -1020,34 +1046,27 @@ out_free:
return ret;
}
-#define deref_locked_genl(p) \
- rcu_dereference_protected(p, lockdep_genl_is_held() && \
- lockdep_is_held(&tcp_metrics_lock))
-
-#define deref_genl(p) rcu_dereference_protected(p, lockdep_genl_is_held())
-
-static int tcp_metrics_flush_all(struct net *net)
+static void tcp_metrics_flush_all(struct net *net)
{
- unsigned int max_rows = 1U << net->ipv4.tcp_metrics_hash_log;
- struct tcpm_hash_bucket *hb = net->ipv4.tcp_metrics_hash;
+ unsigned int max_rows = 1U << tcp_metrics_hash_log;
+ struct tcpm_hash_bucket *hb = tcp_metrics_hash;
struct tcp_metrics_block *tm;
unsigned int row;
for (row = 0; row < max_rows; row++, hb++) {
+ struct tcp_metrics_block __rcu **pp;
spin_lock_bh(&tcp_metrics_lock);
- tm = deref_locked_genl(hb->chain);
- if (tm)
- hb->chain = NULL;
- spin_unlock_bh(&tcp_metrics_lock);
- while (tm) {
- struct tcp_metrics_block *next;
-
- next = deref_genl(tm->tcpm_next);
- kfree_rcu(tm, rcu_head);
- tm = next;
+ pp = &hb->chain;
+ for (tm = deref_locked(*pp); tm; tm = deref_locked(*pp)) {
+ if (net_eq(tm_net(tm), net)) {
+ *pp = tm->tcpm_next;
+ kfree_rcu(tm, rcu_head);
+ } else {
+ pp = &tm->tcpm_next;
+ }
}
+ spin_unlock_bh(&tcp_metrics_lock);
}
- return 0;
}
static int tcp_metrics_nl_cmd_del(struct sk_buff *skb, struct genl_info *info)
@@ -1064,19 +1083,23 @@ static int tcp_metrics_nl_cmd_del(struct sk_buff *skb, struct genl_info *info)
ret = parse_nl_addr(info, &daddr, &hash, 1);
if (ret < 0)
return ret;
- if (ret > 0)
- return tcp_metrics_flush_all(net);
+ if (ret > 0) {
+ tcp_metrics_flush_all(net);
+ return 0;
+ }
ret = parse_nl_saddr(info, &saddr);
if (ret < 0)
src = false;
- hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
- hb = net->ipv4.tcp_metrics_hash + hash;
+ hash ^= net_hash_mix(net);
+ hash = hash_32(hash, tcp_metrics_hash_log);
+ hb = tcp_metrics_hash + hash;
pp = &hb->chain;
spin_lock_bh(&tcp_metrics_lock);
- for (tm = deref_locked_genl(*pp); tm; tm = deref_locked_genl(*pp)) {
+ for (tm = deref_locked(*pp); tm; tm = deref_locked(*pp)) {
if (addr_same(&tm->tcpm_daddr, &daddr) &&
- (!src || addr_same(&tm->tcpm_saddr, &saddr))) {
+ (!src || addr_same(&tm->tcpm_saddr, &saddr)) &&
+ net_eq(tm_net(tm), net)) {
*pp = tm->tcpm_next;
kfree_rcu(tm, rcu_head);
found = true;
@@ -1126,6 +1149,9 @@ static int __net_init tcp_net_metrics_init(struct net *net)
size_t size;
unsigned int slots;
+ if (!net_eq(net, &init_net))
+ return 0;
+
slots = tcpmhash_entries;
if (!slots) {
if (totalram_pages >= 128 * 1024)
@@ -1134,14 +1160,14 @@ static int __net_init tcp_net_metrics_init(struct net *net)
slots = 8 * 1024;
}
- net->ipv4.tcp_metrics_hash_log = order_base_2(slots);
- size = sizeof(struct tcpm_hash_bucket) << net->ipv4.tcp_metrics_hash_log;
+ tcp_metrics_hash_log = order_base_2(slots);
+ size = sizeof(struct tcpm_hash_bucket) << tcp_metrics_hash_log;
- net->ipv4.tcp_metrics_hash = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
- if (!net->ipv4.tcp_metrics_hash)
- net->ipv4.tcp_metrics_hash = vzalloc(size);
+ tcp_metrics_hash = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
+ if (!tcp_metrics_hash)
+ tcp_metrics_hash = vzalloc(size);
- if (!net->ipv4.tcp_metrics_hash)
+ if (!tcp_metrics_hash)
return -ENOMEM;
return 0;
@@ -1149,19 +1175,7 @@ static int __net_init tcp_net_metrics_init(struct net *net)
static void __net_exit tcp_net_metrics_exit(struct net *net)
{
- unsigned int i;
-
- for (i = 0; i < (1U << net->ipv4.tcp_metrics_hash_log) ; i++) {
- struct tcp_metrics_block *tm, *next;
-
- tm = rcu_dereference_protected(net->ipv4.tcp_metrics_hash[i].chain, 1);
- while (tm) {
- next = rcu_dereference_protected(tm->tcpm_next, 1);
- kfree(tm);
- tm = next;
- }
- }
- kvfree(net->ipv4.tcp_metrics_hash);
+ tcp_metrics_flush_all(net);
}
static __net_initdata struct pernet_operations tcp_net_metrics_ops = {
@@ -1175,16 +1189,10 @@ void __init tcp_metrics_init(void)
ret = register_pernet_subsys(&tcp_net_metrics_ops);
if (ret < 0)
- goto cleanup;
+ panic("Could not allocate the tcp_metrics hash table\n");
+
ret = genl_register_family_with_ops(&tcp_metrics_nl_family,
tcp_metrics_nl_ops);
if (ret < 0)
- goto cleanup_subsys;
- return;
-
-cleanup_subsys:
- unregister_pernet_subsys(&tcp_net_metrics_ops);
-
-cleanup:
- return;
+ panic("Could not register tcp_metrics generic netlink\n");
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-26 10:02:11 +0000