unimore/kernel-hacking-2024-linux-steffo
unimore/kernel-hacking-2024-linux-steffo
1
Fork 0
Code Pull requests 5 Wiki Activity

ipv4: Fix input route performance regression.

Browse source 
With the routing cache removal we lost the "noref" code paths on
input, and this can kill some routing workloads.

Reinstate the noref path when we hit a cached route in the FIB
nexthops.

With help from Eric Dumazet.

Reported-by: Alexander Duyck <alexander.duyck@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller 2012-07-26 11:14:38 +00:00
parent 4487e64de6
commit c6cffba4ff
7 changed files with 48 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
include/net/route.h
View file

@ -30,6 +30,7 @@
#include <net/inet_sock.h> #include <net/inet_sock.h>
#include <linux/in_route.h> #include <linux/in_route.h>
#include <linux/rtnetlink.h> #include <linux/rtnetlink.h>
#include <linux/rcupdate.h>
#include <linux/route.h> #include <linux/route.h>
#include <linux/ip.h> #include <linux/ip.h>
#include <linux/cache.h> #include <linux/cache.h>
@ -157,9 +158,23 @@ static inline struct rtable *ip_route_output_gre(struct net *net, struct flowi4
return ip_route_output_key(net, fl4); return ip_route_output_key(net, fl4);
} }
extern int ip_route_input(struct sk_buff *skb, __be32 dst, __be32 src, extern int ip_route_input_noref(struct sk_buff *skb, __be32 dst, __be32 src,
u8 tos, struct net_device *devin); u8 tos, struct net_device *devin);
static inline int ip_route_input(struct sk_buff *skb, __be32 dst, __be32 src,
u8 tos, struct net_device *devin)
{
int err;
rcu_read_lock();
err = ip_route_input_noref(skb, dst, src, tos, devin);
if (!err)
skb_dst_force(skb);
rcu_read_unlock();
return err;
}
extern void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu, extern void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
int oif, u32 mark, u8 protocol, int flow_flags); int oif, u32 mark, u8 protocol, int flow_flags);
extern void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu); extern void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu);

2
net/ipv4/arp.c
View file

@ -827,7 +827,7 @@ static int arp_process(struct sk_buff *skb)
} }
if (arp->ar_op == htons(ARPOP_REQUEST) && if (arp->ar_op == htons(ARPOP_REQUEST) &&
ip_route_input(skb, tip, sip, 0, dev) == 0) { ip_route_input_noref(skb, tip, sip, 0, dev) == 0) {
rt = skb_rtable(skb); rt = skb_rtable(skb);
addr_type = rt->rt_type; addr_type = rt->rt_type;

4
net/ipv4/fib_semantics.c
View file

@ -172,9 +172,9 @@ static void free_fib_info_rcu(struct rcu_head *head)
if (nexthop_nh->nh_exceptions) if (nexthop_nh->nh_exceptions)
free_nh_exceptions(nexthop_nh); free_nh_exceptions(nexthop_nh);
if (nexthop_nh->nh_rth_output) if (nexthop_nh->nh_rth_output)
dst_release(&nexthop_nh->nh_rth_output->dst); dst_free(&nexthop_nh->nh_rth_output->dst);
if (nexthop_nh->nh_rth_input) if (nexthop_nh->nh_rth_input)
dst_release(&nexthop_nh->nh_rth_input->dst); dst_free(&nexthop_nh->nh_rth_input->dst);
} endfor_nexthops(fi); } endfor_nexthops(fi);
release_net(fi->fib_net); release_net(fi->fib_net);

2
net/ipv4/ip_fragment.c
View file

@ -258,7 +258,7 @@ static void ip_expire(unsigned long arg)
/* skb dst is stale, drop it, and perform route lookup again */ /* skb dst is stale, drop it, and perform route lookup again */
skb_dst_drop(head); skb_dst_drop(head);
iph = ip_hdr(head); iph = ip_hdr(head);
err = ip_route_input(head, iph->daddr, iph->saddr, err = ip_route_input_noref(head, iph->daddr, iph->saddr,
iph->tos, head->dev); iph->tos, head->dev);
if (err) if (err)
goto out_rcu_unlock; goto out_rcu_unlock;

2
net/ipv4/ip_input.c
View file

@ -339,7 +339,7 @@ static int ip_rcv_finish(struct sk_buff *skb)
* how the packet travels inside Linux networking. * how the packet travels inside Linux networking.
*/ */
if (!skb_dst(skb)) { if (!skb_dst(skb)) {
int err = ip_route_input(skb, iph->daddr, iph->saddr, int err = ip_route_input_noref(skb, iph->daddr, iph->saddr,
iph->tos, skb->dev); iph->tos, skb->dev);
if (unlikely(err)) { if (unlikely(err)) {
if (err == -EXDEV) if (err == -EXDEV)

46
net/ipv4/route.c
View file

@ -1199,10 +1199,9 @@ static void rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
fnhe->fnhe_stamp = jiffies; fnhe->fnhe_stamp = jiffies;
} }
static inline void rt_release_rcu(struct rcu_head *head) static inline void rt_free(struct rtable *rt)
{ {
struct dst_entry *dst = container_of(head, struct dst_entry, rcu_head); call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
dst_release(dst);
} }
static void rt_cache_route(struct fib_nh *nh, struct rtable *rt) static void rt_cache_route(struct fib_nh *nh, struct rtable *rt)
@ -1216,9 +1215,15 @@ static void rt_cache_route(struct fib_nh *nh, struct rtable *rt)
prev = cmpxchg(p, orig, rt); prev = cmpxchg(p, orig, rt);
if (prev == orig) { if (prev == orig) {
dst_clone(&rt->dst);
if (orig) if (orig)
call_rcu_bh(&orig->dst.rcu_head, rt_release_rcu); rt_free(orig);
} else {
/* Routes we intend to cache in the FIB nexthop have
* the DST_NOCACHE bit clear. However, if we are
* unsuccessful at storing this route into the cache
* we really need to set it.
*/
rt->dst.flags |= DST_NOCACHE;
} }
} }
@ -1245,7 +1250,7 @@ static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
#ifdef CONFIG_IP_ROUTE_CLASSID #ifdef CONFIG_IP_ROUTE_CLASSID
rt->dst.tclassid = nh->nh_tclassid; rt->dst.tclassid = nh->nh_tclassid;
#endif #endif
if (!(rt->dst.flags & DST_HOST)) if (!(rt->dst.flags & DST_NOCACHE))
rt_cache_route(nh, rt); rt_cache_route(nh, rt);
} }
@ -1261,7 +1266,7 @@ static struct rtable *rt_dst_alloc(struct net_device *dev,
bool nopolicy, bool noxfrm, bool will_cache) bool nopolicy, bool noxfrm, bool will_cache)
{ {
return dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK, return dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
(will_cache ? 0 : DST_HOST) | DST_NOCACHE | (will_cache ? 0 : (DST_HOST | DST_NOCACHE)) |
(nopolicy ? DST_NOPOLICY : 0) | (nopolicy ? DST_NOPOLICY : 0) |
(noxfrm ? DST_NOXFRM : 0)); (noxfrm ? DST_NOXFRM : 0));
} }
@ -1366,8 +1371,7 @@ static void ip_handle_martian_source(struct net_device *dev,
static int __mkroute_input(struct sk_buff *skb, static int __mkroute_input(struct sk_buff *skb,
const struct fib_result *res, const struct fib_result *res,
struct in_device *in_dev, struct in_device *in_dev,
__be32 daddr, __be32 saddr, u32 tos, __be32 daddr, __be32 saddr, u32 tos)
struct rtable **result)
{ {
struct rtable *rth; struct rtable *rth;
int err; int err;
@ -1418,7 +1422,7 @@ static int __mkroute_input(struct sk_buff *skb,
if (!itag) { if (!itag) {
rth = FIB_RES_NH(*res).nh_rth_input; rth = FIB_RES_NH(*res).nh_rth_input;
if (rt_cache_valid(rth)) { if (rt_cache_valid(rth)) {
dst_hold(&rth->dst); skb_dst_set_noref(skb, &rth->dst);
goto out; goto out;
} }
do_cache = true; do_cache = true;
@ -1445,8 +1449,8 @@ static int __mkroute_input(struct sk_buff *skb,
rth->dst.output = ip_output; rth->dst.output = ip_output;
rt_set_nexthop(rth, daddr, res, NULL, res->fi, res->type, itag); rt_set_nexthop(rth, daddr, res, NULL, res->fi, res->type, itag);
skb_dst_set(skb, &rth->dst);
out: out:
*result = rth;
err = 0; err = 0;
cleanup: cleanup:
return err; return err;
@ -1458,21 +1462,13 @@ static int ip_mkroute_input(struct sk_buff *skb,
struct in_device *in_dev, struct in_device *in_dev,
__be32 daddr, __be32 saddr, u32 tos) __be32 daddr, __be32 saddr, u32 tos)
{ {
struct rtable *rth = NULL;
int err;
#ifdef CONFIG_IP_ROUTE_MULTIPATH #ifdef CONFIG_IP_ROUTE_MULTIPATH
if (res->fi && res->fi->fib_nhs > 1) if (res->fi && res->fi->fib_nhs > 1)
fib_select_multipath(res); fib_select_multipath(res);
#endif #endif
/* create a routing cache entry */ /* create a routing cache entry */
err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth); return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
if (err)
return err;
skb_dst_set(skb, &rth->dst);
return 0;
} }
/* /*
@ -1588,8 +1584,9 @@ out: return err;
if (!itag) { if (!itag) {
rth = FIB_RES_NH(res).nh_rth_input; rth = FIB_RES_NH(res).nh_rth_input;
if (rt_cache_valid(rth)) { if (rt_cache_valid(rth)) {
dst_hold(&rth->dst); skb_dst_set_noref(skb, &rth->dst);
goto set_and_out; err = 0;
goto out;
} }
do_cache = true; do_cache = true;
} }
@ -1620,7 +1617,6 @@ out: return err;
} }
if (do_cache) if (do_cache)
rt_cache_route(&FIB_RES_NH(res), rth); rt_cache_route(&FIB_RES_NH(res), rth);
set_and_out:
skb_dst_set(skb, &rth->dst); skb_dst_set(skb, &rth->dst);
err = 0; err = 0;
goto out; goto out;
@ -1658,7 +1654,7 @@ out: return err;
goto out; goto out;
} }
int ip_route_input(struct sk_buff *skb, __be32 daddr, __be32 saddr, int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
u8 tos, struct net_device *dev) u8 tos, struct net_device *dev)
{ {
int res; int res;
@ -1702,7 +1698,7 @@ int ip_route_input(struct sk_buff *skb, __be32 daddr, __be32 saddr,
rcu_read_unlock(); rcu_read_unlock();
return res; return res;
} }
EXPORT_SYMBOL(ip_route_input); EXPORT_SYMBOL(ip_route_input_noref);
/* called with rcu_read_lock() */ /* called with rcu_read_lock() */
static struct rtable *__mkroute_output(const struct fib_result *res, static struct rtable *__mkroute_output(const struct fib_result *res,

2
net/ipv4/xfrm4_input.c
View file

@ -27,7 +27,7 @@ static inline int xfrm4_rcv_encap_finish(struct sk_buff *skb)
if (skb_dst(skb) == NULL) { if (skb_dst(skb) == NULL) {
const struct iphdr *iph = ip_hdr(skb); const struct iphdr *iph = ip_hdr(skb);
if (ip_route_input(skb, iph->daddr, iph->saddr, if (ip_route_input_noref(skb, iph->daddr, iph->saddr,
iph->tos, skb->dev)) iph->tos, skb->dev))
goto drop; goto drop;
} }