int total;
};
+#define INETPEER_BASE_BIT 0x1UL
+
+static inline struct inet_peer *inetpeer_ptr(unsigned long val)
+{
+ BUG_ON(val & INETPEER_BASE_BIT);
+ return (struct inet_peer *) val;
+}
+
+static inline struct inet_peer_base *inetpeer_base_ptr(unsigned long val)
+{
+ if (!(val & INETPEER_BASE_BIT))
+ return NULL;
+ val &= ~INETPEER_BASE_BIT;
+ return (struct inet_peer_base *) val;
+}
+
+static inline bool inetpeer_ptr_is_peer(unsigned long val)
+{
+ return !(val & INETPEER_BASE_BIT);
+}
+
+static inline void __inetpeer_ptr_set_peer(unsigned long *val, struct inet_peer *peer)
+{
+ /* This implicitly clears INETPEER_BASE_BIT */
+ *val = (unsigned long) peer;
+}
+
+static inline bool inetpeer_ptr_set_peer(unsigned long *ptr, struct inet_peer *peer)
+{
+ unsigned long val = (unsigned long) peer;
+ unsigned long orig = *ptr;
+
+ if (!(orig & INETPEER_BASE_BIT) || !val ||
+ cmpxchg(ptr, orig, val) != orig)
+ return false;
+ return true;
+}
+
+static inline void inetpeer_init_ptr(unsigned long *ptr, struct inet_peer_base *base)
+{
+ *ptr = (unsigned long) base | INETPEER_BASE_BIT;
+}
+
+static inline void inetpeer_transfer_peer(unsigned long *to, unsigned long *from)
+{
+ unsigned long val = *from;
+
+ *to = val;
+ if (inetpeer_ptr_is_peer(val)) {
+ struct inet_peer *peer = inetpeer_ptr(val);
+ atomic_inc(&peer->refcnt);
+ }
+}
+
extern void inet_peer_base_init(struct inet_peer_base *);
void inet_initpeers(void) __init;
u32 rt6i_peer_genid;
struct inet6_dev *rt6i_idev;
- struct inet_peer *rt6i_peer;
+ unsigned long _rt6i_peer;
#ifdef CONFIG_XFRM
u32 rt6i_flow_cache_genid;
u8 rt6i_protocol;
};
+static inline struct inet_peer *rt6_peer_ptr(struct rt6_info *rt)
+{
+ return inetpeer_ptr(rt->_rt6i_peer);
+}
+
+static inline bool rt6_has_peer(struct rt6_info *rt)
+{
+ return inetpeer_ptr_is_peer(rt->_rt6i_peer);
+}
+
+static inline void __rt6_set_peer(struct rt6_info *rt, struct inet_peer *peer)
+{
+ __inetpeer_ptr_set_peer(&rt->_rt6i_peer, peer);
+}
+
+static inline bool rt6_set_peer(struct rt6_info *rt, struct inet_peer *peer)
+{
+ return inetpeer_ptr_set_peer(&rt->_rt6i_peer, peer);
+}
+
+static inline void rt6_init_peer(struct rt6_info *rt, struct inet_peer_base *base)
+{
+ inetpeer_init_ptr(&rt->_rt6i_peer, base);
+}
+
+static inline void rt6_transfer_peer(struct rt6_info *rt, struct rt6_info *ort)
+{
+ inetpeer_transfer_peer(&rt->_rt6i_peer, &ort->_rt6i_peer);
+}
+
static inline struct inet6_dev *ip6_dst_idev(struct dst_entry *dst)
{
return ((struct rt6_info *)dst)->rt6i_idev;
static inline struct inet_peer *__rt6_get_peer(struct rt6_info *rt, int create)
{
- if (rt->rt6i_peer)
- return rt->rt6i_peer;
+ if (rt6_has_peer(rt))
+ return rt6_peer_ptr(rt);
rt6_bind_peer(rt, create);
- return rt->rt6i_peer;
+ return rt6_peer_ptr(rt);
}
static inline struct inet_peer *rt6_get_peer(struct rt6_info *rt)
/* Miscellaneous cached information */
__be32 rt_spec_dst; /* RFC1122 specific destination */
u32 rt_peer_genid;
- struct inet_peer *peer; /* long-living peer info */
+ unsigned long _peer; /* long-living peer info */
struct fib_info *fi; /* for client ref to shared metrics */
};
+static inline struct inet_peer *rt_peer_ptr(struct rtable *rt)
+{
+ return inetpeer_ptr(rt->_peer);
+}
+
+static inline bool rt_has_peer(struct rtable *rt)
+{
+ return inetpeer_ptr_is_peer(rt->_peer);
+}
+
+static inline void __rt_set_peer(struct rtable *rt, struct inet_peer *peer)
+{
+ __inetpeer_ptr_set_peer(&rt->_peer, peer);
+}
+
+static inline bool rt_set_peer(struct rtable *rt, struct inet_peer *peer)
+{
+ return inetpeer_ptr_set_peer(&rt->_peer, peer);
+}
+
+static inline void rt_init_peer(struct rtable *rt, struct inet_peer_base *base)
+{
+ inetpeer_init_ptr(&rt->_peer, base);
+}
+
+static inline void rt_transfer_peer(struct rtable *rt, struct rtable *ort)
+{
+ rt->_peer = ort->_peer;
+ if (rt_has_peer(ort)) {
+ struct inet_peer *peer = rt_peer_ptr(ort);
+ atomic_inc(&peer->refcnt);
+ }
+}
+
static inline bool rt_is_input_route(const struct rtable *rt)
{
return rt->rt_route_iif != 0;
static inline struct inet_peer *__rt_get_peer(struct rtable *rt, __be32 daddr, int create)
{
- if (rt->peer)
- return rt->peer;
+ if (rt_has_peer(rt))
+ return rt_peer_ptr(rt);
rt_bind_peer(rt, daddr, create);
- return rt->peer;
+ return rt_peer_ptr(rt);
}
static inline struct inet_peer *rt_get_peer(struct rtable *rt, __be32 daddr)
static inline int rt_valuable(struct rtable *rth)
{
return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
- (rth->peer && rth->peer->pmtu_expires);
+ (rt_has_peer(rth) && rt_peer_ptr(rth)->pmtu_expires);
}
static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
void rt_bind_peer(struct rtable *rt, __be32 daddr, int create)
{
- struct net *net = dev_net(rt->dst.dev);
+ struct inet_peer_base *base;
struct inet_peer *peer;
- peer = inet_getpeer_v4(net->ipv4.peers, daddr, create);
+ base = inetpeer_base_ptr(rt->_peer);
+ if (!base)
+ return;
+
+ peer = inet_getpeer_v4(base, daddr, create);
- if (peer && cmpxchg(&rt->peer, NULL, peer) != NULL)
+ if (!rt_set_peer(rt, peer))
inet_putpeer(peer);
else
rt->rt_peer_genid = rt_peer_genid();
rt_genid(dev_net(dst->dev)));
rt_del(hash, rt);
ret = NULL;
- } else if (rt->peer && peer_pmtu_expired(rt->peer)) {
- dst_metric_set(dst, RTAX_MTU, rt->peer->pmtu_orig);
+ } else if (rt_has_peer(rt)) {
+ struct inet_peer *peer = rt_peer_ptr(rt);
+ if (peer_pmtu_expired(peer))
+ dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
}
}
return ret;
static void ipv4_dst_destroy(struct dst_entry *dst)
{
struct rtable *rt = (struct rtable *) dst;
- struct inet_peer *peer = rt->peer;
if (rt->fi) {
fib_info_put(rt->fi);
rt->fi = NULL;
}
- if (peer) {
- rt->peer = NULL;
+ if (rt_has_peer(rt)) {
+ struct inet_peer *peer = rt_peer_ptr(rt);
inet_putpeer(peer);
}
}
icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
rt = skb_rtable(skb);
- if (rt && rt->peer && peer_pmtu_cleaned(rt->peer))
- dst_metric_set(&rt->dst, RTAX_MTU, rt->peer->pmtu_orig);
+ if (rt && rt_has_peer(rt)) {
+ struct inet_peer *peer = rt_peer_ptr(rt);
+ if (peer_pmtu_cleaned(peer))
+ dst_metric_set(&rt->dst, RTAX_MTU, peer->pmtu_orig);
+ }
}
static int ip_rt_bug(struct sk_buff *skb)
static void rt_init_metrics(struct rtable *rt, const struct flowi4 *fl4,
struct fib_info *fi)
{
- struct net *net = dev_net(rt->dst.dev);
+ struct inet_peer_base *base;
struct inet_peer *peer;
int create = 0;
if (fl4 && (fl4->flowi4_flags & FLOWI_FLAG_PRECOW_METRICS))
create = 1;
- rt->peer = peer = inet_getpeer_v4(net->ipv4.peers, rt->rt_dst, create);
+ base = inetpeer_base_ptr(rt->_peer);
+ BUG_ON(!base);
+
+ peer = inet_getpeer_v4(base, rt->rt_dst, create);
if (peer) {
+ __rt_set_peer(rt, peer);
rt->rt_peer_genid = rt_peer_genid();
if (inet_metrics_new(peer))
memcpy(peer->metrics, fi->fib_metrics,
rth->rt_gateway = daddr;
rth->rt_spec_dst= spec_dst;
rth->rt_peer_genid = 0;
- rth->peer = NULL;
+ rt_init_peer(rth, dev_net(dev)->ipv4.peers);
rth->fi = NULL;
if (our) {
rth->dst.input= ip_local_deliver;
rth->rt_gateway = daddr;
rth->rt_spec_dst= spec_dst;
rth->rt_peer_genid = 0;
- rth->peer = NULL;
+ rt_init_peer(rth, dev_net(rth->dst.dev)->ipv4.peers);
rth->fi = NULL;
rth->dst.input = ip_forward;
rth->rt_gateway = daddr;
rth->rt_spec_dst= spec_dst;
rth->rt_peer_genid = 0;
- rth->peer = NULL;
+ rt_init_peer(rth, net->ipv4.peers);
rth->fi = NULL;
if (res.type == RTN_UNREACHABLE) {
rth->dst.input= ip_error;
rth->rt_gateway = fl4->daddr;
rth->rt_spec_dst= fl4->saddr;
rth->rt_peer_genid = 0;
- rth->peer = NULL;
+ rt_init_peer(rth, dev_net(dev_out)->ipv4.peers);
rth->fi = NULL;
RT_CACHE_STAT_INC(out_slow_tot);
rt->rt_src = ort->rt_src;
rt->rt_gateway = ort->rt_gateway;
rt->rt_spec_dst = ort->rt_spec_dst;
- rt->peer = ort->peer;
- if (rt->peer)
- atomic_inc(&rt->peer->refcnt);
+ rt_transfer_peer(rt, ort);
rt->fi = ort->fi;
if (rt->fi)
atomic_inc(&rt->fi->fib_clntref);
struct rtmsg *r;
struct nlmsghdr *nlh;
unsigned long expires = 0;
- const struct inet_peer *peer = rt->peer;
u32 id = 0, ts = 0, tsage = 0, error;
nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
goto nla_put_failure;
error = rt->dst.error;
- if (peer) {
- inet_peer_refcheck(rt->peer);
+ if (rt_has_peer(rt)) {
+ const struct inet_peer *peer = rt_peer_ptr(rt);
+ inet_peer_refcheck(peer);
id = atomic_read(&peer->ip_id_count) & 0xffff;
if (peer->tcp_ts_stamp) {
ts = peer->tcp_ts;
xdst->u.dst.dev = dev;
dev_hold(dev);
- xdst->u.rt.peer = rt->peer;
- if (rt->peer)
- atomic_inc(&rt->peer->refcnt);
+ rt_transfer_peer(&xdst->u.rt, rt);
/* Sheit... I remember I did this right. Apparently,
* it was magically lost, so this code needs audit */
dst_destroy_metrics_generic(dst);
- if (likely(xdst->u.rt.peer))
- inet_putpeer(xdst->u.rt.peer);
+ if (rt_has_peer(&xdst->u.rt)) {
+ struct inet_peer *peer = rt_peer_ptr(&xdst->u.rt);
+ inet_putpeer(peer);
+ }
xfrm_dst_destroy(xdst);
}
#endif
/* allocate dst with ip6_dst_ops */
-static inline struct rt6_info *ip6_dst_alloc(struct dst_ops *ops,
+static inline struct rt6_info *ip6_dst_alloc(struct net *net,
struct net_device *dev,
int flags)
{
- struct rt6_info *rt = dst_alloc(ops, dev, 0, 0, flags);
+ struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev,
+ 0, 0, flags);
- if (rt)
+ if (rt) {
memset(&rt->rt6i_table, 0,
sizeof(*rt) - sizeof(struct dst_entry));
-
+ rt6_init_peer(rt, net->ipv6.peers);
+ }
return rt;
}
{
struct rt6_info *rt = (struct rt6_info *)dst;
struct inet6_dev *idev = rt->rt6i_idev;
- struct inet_peer *peer = rt->rt6i_peer;
if (!(rt->dst.flags & DST_HOST))
dst_destroy_metrics_generic(dst);
if (!(rt->rt6i_flags & RTF_EXPIRES) && dst->from)
dst_release(dst->from);
- if (peer) {
- rt->rt6i_peer = NULL;
+ if (rt6_has_peer(rt)) {
+ struct inet_peer *peer = rt6_peer_ptr(rt);
inet_putpeer(peer);
}
}
void rt6_bind_peer(struct rt6_info *rt, int create)
{
- struct net *net = dev_net(rt->dst.dev);
+ struct inet_peer_base *base;
struct inet_peer *peer;
- peer = inet_getpeer_v6(net->ipv6.peers, &rt->rt6i_dst.addr, create);
- if (peer && cmpxchg(&rt->rt6i_peer, NULL, peer) != NULL)
+ base = inetpeer_base_ptr(rt->_rt6i_peer);
+ if (!base)
+ return;
+
+ peer = inet_getpeer_v6(base, &rt->rt6i_dst.addr, create);
+ if (!rt6_set_peer(rt, peer))
inet_putpeer(peer);
else
rt->rt6i_peer_genid = rt6_peer_genid();
rt = dst_alloc(&ip6_dst_blackhole_ops, ort->dst.dev, 1, 0, 0);
if (rt) {
memset(&rt->rt6i_table, 0, sizeof(*rt) - sizeof(struct dst_entry));
+ rt6_init_peer(rt, net->ipv6.peers);
new = &rt->dst;
if (rt->rt6i_node && (rt->rt6i_node->fn_sernum == cookie)) {
if (rt->rt6i_peer_genid != rt6_peer_genid()) {
- if (!rt->rt6i_peer)
+ if (!rt6_has_peer(rt))
rt6_bind_peer(rt, 0);
rt->rt6i_peer_genid = rt6_peer_genid();
}
if (unlikely(!idev))
return ERR_PTR(-ENODEV);
- rt = ip6_dst_alloc(&net->ipv6.ip6_dst_ops, dev, 0);
+ rt = ip6_dst_alloc(net, dev, 0);
if (unlikely(!rt)) {
in6_dev_put(idev);
dst = ERR_PTR(-ENOMEM);
if (!table)
goto out;
- rt = ip6_dst_alloc(&net->ipv6.ip6_dst_ops, NULL, DST_NOCOUNT);
+ rt = ip6_dst_alloc(net, NULL, DST_NOCOUNT);
if (!rt) {
err = -ENOMEM;
const struct in6_addr *dest)
{
struct net *net = dev_net(ort->dst.dev);
- struct rt6_info *rt = ip6_dst_alloc(&net->ipv6.ip6_dst_ops,
- ort->dst.dev, 0);
+ struct rt6_info *rt = ip6_dst_alloc(net, ort->dst.dev, 0);
if (rt) {
rt->dst.input = ort->dst.input;
bool anycast)
{
struct net *net = dev_net(idev->dev);
- struct rt6_info *rt = ip6_dst_alloc(&net->ipv6.ip6_dst_ops,
- net->loopback_dev, 0);
+ struct rt6_info *rt = ip6_dst_alloc(net, net->loopback_dev, 0);
int err;
if (!rt) {
else
expires = INT_MAX;
- peer = rt->rt6i_peer;
+ peer = NULL;
+ if (rt6_has_peer(rt))
+ peer = rt6_peer_ptr(rt);
ts = tsage = 0;
if (peer && peer->tcp_ts_stamp) {
ts = peer->tcp_ts;
if (!xdst->u.rt6.rt6i_idev)
return -ENODEV;
- xdst->u.rt6.rt6i_peer = rt->rt6i_peer;
- if (rt->rt6i_peer)
- atomic_inc(&rt->rt6i_peer->refcnt);
+ rt6_transfer_peer(&xdst->u.rt6, rt);
/* Sheit... I remember I did this right. Apparently,
* it was magically lost, so this code needs audit */
if (likely(xdst->u.rt6.rt6i_idev))
in6_dev_put(xdst->u.rt6.rt6i_idev);
dst_destroy_metrics_generic(dst);
- if (likely(xdst->u.rt6.rt6i_peer))
- inet_putpeer(xdst->u.rt6.rt6i_peer);
+ if (rt6_has_peer(&xdst->u.rt6)) {
+ struct inet_peer *peer = rt6_peer_ptr(&xdst->u.rt6);
+ inet_putpeer(peer);
+ }
xfrm_dst_destroy(xdst);
}