jenna
/
kernel_samsung_sm7125
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

udp: Use hlist_nulls in UDP RCU code

Browse Source 
This is a straightforward patch, using hlist_nulls infrastructure.

RCUification already done on UDP two weeks ago.

Using hlist_nulls permits us to avoid some memory barriers, both
at lookup time and delete time.

Patch is large because it adds new macros to include/net/sock.h.
These macros will be used by TCP & DCCP in next patch.

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
tirimbino
Eric Dumazet 16 years ago committed by David S. Miller
parent bbaffaca48
commit 88ab1932ea
5 changed files with 83 additions and 66 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 17
      include/linux/rculist.h
  2. 57
      include/net/sock.h
  3. 2
      include/net/udp.h
  4. 47
      net/ipv4/udp.c
  5. 26
      net/ipv6/udp.c

17
include/linux/rculist.h
Unescape View File

@ -383,22 +383,5 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \
pos = rcu_dereference(pos->next))
/**
* hlist_for_each_entry_rcu_safenext - iterate over rcu list of given type
* @tpos: the type * to use as a loop cursor.
* @pos: the &struct hlist_node to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the hlist_node within the struct.
* @next: the &struct hlist_node to use as a next cursor
*
* Special version of hlist_for_each_entry_rcu that make sure
* each next pointer is fetched before each iteration.
*/
#define hlist_for_each_entry_rcu_safenext(tpos, pos, head, member, next) \
for (pos = rcu_dereference((head)->first); \
pos && ({ next = pos->next; smp_rmb(); prefetch(next); 1; }) && \
({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \
pos = rcu_dereference(next))
#endif /* __KERNEL__ */
#endif

57
include/net/sock.h
Unescape View File

@ -42,6 +42,7 @@
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/list_nulls.h>
#include <linux/timer.h>
#include <linux/cache.h>
#include <linux/module.h>
@ -52,6 +53,7 @@
#include <linux/security.h>
#include <linux/filter.h>
#include <linux/rculist_nulls.h>
#include <asm/atomic.h>
#include <net/dst.h>
@ -106,6 +108,7 @@ struct net;
* @skc_reuse: %SO_REUSEADDR setting
* @skc_bound_dev_if: bound device index if != 0
* @skc_node: main hash linkage for various protocol lookup tables
* @skc_nulls_node: main hash linkage for UDP/UDP-Lite protocol
* @skc_bind_node: bind hash linkage for various protocol lookup tables
* @skc_refcnt: reference count
* @skc_hash: hash value used with various protocol lookup tables
@ -120,7 +123,10 @@ struct sock_common {
volatile unsigned char skc_state;
unsigned char skc_reuse;
int skc_bound_dev_if;
struct hlist_node skc_node;
union {
struct hlist_node skc_node;
struct hlist_nulls_node skc_nulls_node;
};
struct hlist_node skc_bind_node;
atomic_t skc_refcnt;
unsigned int skc_hash;
@ -206,6 +212,7 @@ struct sock {
#define sk_reuse __sk_common.skc_reuse
#define sk_bound_dev_if __sk_common.skc_bound_dev_if
#define sk_node __sk_common.skc_node
#define sk_nulls_node __sk_common.skc_nulls_node
#define sk_bind_node __sk_common.skc_bind_node
#define sk_refcnt __sk_common.skc_refcnt
#define sk_hash __sk_common.skc_hash
@ -300,12 +307,30 @@ static inline struct sock *sk_head(const struct hlist_head *head)
return hlist_empty(head) ? NULL : __sk_head(head);
}
static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
{
return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
}
static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
{
return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
}
static inline struct sock *sk_next(const struct sock *sk)
{
return sk->sk_node.next ?
hlist_entry(sk->sk_node.next, struct sock, sk_node) : NULL;
}
static inline struct sock *sk_nulls_next(const struct sock *sk)
{
return (!is_a_nulls(sk->sk_nulls_node.next)) ?
hlist_nulls_entry(sk->sk_nulls_node.next,
struct sock, sk_nulls_node) :
NULL;
}
static inline int sk_unhashed(const struct sock *sk)
{
return hlist_unhashed(&sk->sk_node);
@ -321,6 +346,11 @@ static __inline__ void sk_node_init(struct hlist_node *node)
node->pprev = NULL;
}
static __inline__ void sk_nulls_node_init(struct hlist_nulls_node *node)
{
node->pprev = NULL;
}
static __inline__ void __sk_del_node(struct sock *sk)
{
__hlist_del(&sk->sk_node);
@ -367,18 +397,18 @@ static __inline__ int sk_del_node_init(struct sock *sk)
return rc;
}
static __inline__ int __sk_del_node_init_rcu(struct sock *sk)
static __inline__ int __sk_nulls_del_node_init_rcu(struct sock *sk)
{
if (sk_hashed(sk)) {
hlist_del_init_rcu(&sk->sk_node);
hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
return 1;
}
return 0;
}
static __inline__ int sk_del_node_init_rcu(struct sock *sk)
static __inline__ int sk_nulls_del_node_init_rcu(struct sock *sk)
{
int rc = __sk_del_node_init_rcu(sk);
int rc = __sk_nulls_del_node_init_rcu(sk);
if (rc) {
/* paranoid for a while -acme */
@ -399,15 +429,15 @@ static __inline__ void sk_add_node(struct sock *sk, struct hlist_head *list)
__sk_add_node(sk, list);
}
static __inline__ void __sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
static __inline__ void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
{
hlist_add_head_rcu(&sk->sk_node, list);
hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
}
static __inline__ void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
static __inline__ void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
{
sock_hold(sk);
__sk_add_node_rcu(sk, list);
__sk_nulls_add_node_rcu(sk, list);
}
static __inline__ void __sk_del_bind_node(struct sock *sk)
@ -423,11 +453,16 @@ static __inline__ void sk_add_bind_node(struct sock *sk,
#define sk_for_each(__sk, node, list) \
hlist_for_each_entry(__sk, node, list, sk_node)
#define sk_for_each_rcu_safenext(__sk, node, list, next) \
hlist_for_each_entry_rcu_safenext(__sk, node, list, sk_node, next)
#define sk_nulls_for_each(__sk, node, list) \
hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
#define sk_nulls_for_each_rcu(__sk, node, list) \
hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
#define sk_for_each_from(__sk, node) \
if (__sk && ({ node = &(__sk)->sk_node; 1; })) \
hlist_for_each_entry_from(__sk, node, sk_node)
#define sk_nulls_for_each_from(__sk, node) \
if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
#define sk_for_each_continue(__sk, node) \
if (__sk && ({ node = &(__sk)->sk_node; 1; })) \
hlist_for_each_entry_continue(__sk, node, sk_node)

2
include/net/udp.h
Unescape View File

@ -51,7 +51,7 @@ struct udp_skb_cb {
#define UDP_SKB_CB(__skb) ((struct udp_skb_cb *)((__skb)->cb))
struct udp_hslot {
struct hlist_head head;
struct hlist_nulls_head head;
spinlock_t lock;
} __attribute__((aligned(2 * sizeof(long))));
struct udp_table {

47
net/ipv4/udp.c
Unescape View File

@ -127,9 +127,9 @@ static int udp_lib_lport_inuse(struct net *net, __u16 num,
const struct sock *sk2))
{
struct sock *sk2;
struct hlist_node *node;
struct hlist_nulls_node *node;
sk_for_each(sk2, node, &hslot->head)
sk_nulls_for_each(sk2, node, &hslot->head)
if (net_eq(sock_net(sk2), net) &&
sk2 != sk &&
sk2->sk_hash == num &&
@ -189,12 +189,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum,
inet_sk(sk)->num = snum;
sk->sk_hash = snum;
if (sk_unhashed(sk)) {
/*
* We need that previous write to sk->sk_hash committed
* before write to sk->next done in following add_node() variant
*/
smp_wmb();
sk_add_node_rcu(sk, &hslot->head);
sk_nulls_add_node_rcu(sk, &hslot->head);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
}
error = 0;
@ -261,7 +256,7 @@ static struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
int dif, struct udp_table *udptable)
{
struct sock *sk, *result;
struct hlist_node *node, *next;
struct hlist_nulls_node *node;
unsigned short hnum = ntohs(dport);
unsigned int hash = udp_hashfn(net, hnum);
struct udp_hslot *hslot = &udptable->hash[hash];
@ -271,13 +266,7 @@ static struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
begin:
result = NULL;
badness = -1;
sk_for_each_rcu_safenext(sk, node, &hslot->head, next) {
/*
* lockless reader, and SLAB_DESTROY_BY_RCU items:
* We must check this item was not moved to another chain
*/
if (udp_hashfn(net, sk->sk_hash) != hash)
goto begin;
sk_nulls_for_each_rcu(sk, node, &hslot->head) {
score = compute_score(sk, net, saddr, hnum, sport,
daddr, dport, dif);
if (score > badness) {
@ -285,6 +274,14 @@ begin:
badness = score;
}
}
/*
* if the nulls value we got at the end of this lookup is
* not the expected one, we must restart lookup.
* We probably met an item that was moved to another chain.
*/
if (get_nulls_value(node) != hash)
goto begin;
if (result) {
if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
result = NULL;
@ -325,11 +322,11 @@ static inline struct sock *udp_v4_mcast_next(struct net *net, struct sock *sk,
__be16 rmt_port, __be32 rmt_addr,
int dif)
{
struct hlist_node *node;
struct hlist_nulls_node *node;
struct sock *s = sk;
unsigned short hnum = ntohs(loc_port);
sk_for_each_from(s, node) {
sk_nulls_for_each_from(s, node) {
struct inet_sock *inet = inet_sk(s);
if (!net_eq(sock_net(s), net) ||
@ -977,7 +974,7 @@ void udp_lib_unhash(struct sock *sk)
struct udp_hslot *hslot = &udptable->hash[hash];
spin_lock_bh(&hslot->lock);
if (sk_del_node_init_rcu(sk)) {
if (sk_nulls_del_node_init_rcu(sk)) {
inet_sk(sk)->num = 0;
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
}
@ -1130,7 +1127,7 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
int dif;
spin_lock(&hslot->lock);
sk = sk_head(&hslot->head);
sk = sk_nulls_head(&hslot->head);
dif = skb->dev->ifindex;
sk = udp_v4_mcast_next(net, sk, uh->dest, daddr, uh->source, saddr, dif);
if (sk) {
@ -1139,7 +1136,7 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
do {
struct sk_buff *skb1 = skb;
sknext = udp_v4_mcast_next(net, sk_next(sk), uh->dest,
sknext = udp_v4_mcast_next(net, sk_nulls_next(sk), uh->dest,
daddr, uh->source, saddr,
dif);
if (sknext)
@ -1560,10 +1557,10 @@ static struct sock *udp_get_first(struct seq_file *seq, int start)
struct net *net = seq_file_net(seq);
for (state->bucket = start; state->bucket < UDP_HTABLE_SIZE; ++state->bucket) {
struct hlist_node *node;
struct hlist_nulls_node *node;
struct udp_hslot *hslot = &state->udp_table->hash[state->bucket];
spin_lock_bh(&hslot->lock);
sk_for_each(sk, node, &hslot->head) {
sk_nulls_for_each(sk, node, &hslot->head) {
if (!net_eq(sock_net(sk), net))
continue;
if (sk->sk_family == state->family)
@ -1582,7 +1579,7 @@ static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
struct net *net = seq_file_net(seq);
do {
sk = sk_next(sk);
sk = sk_nulls_next(sk);
} while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != state->family));
if (!sk) {
@ -1753,7 +1750,7 @@ void __init udp_table_init(struct udp_table *table)
int i;
for (i = 0; i < UDP_HTABLE_SIZE; i++) {
INIT_HLIST_HEAD(&table->hash[i].head);
INIT_HLIST_NULLS_HEAD(&table->hash[i].head, i);
spin_lock_init(&table->hash[i].lock);
}
}

26
net/ipv6/udp.c
Unescape View File

@ -98,7 +98,7 @@ static struct sock *__udp6_lib_lookup(struct net *net,
int dif, struct udp_table *udptable)
{
struct sock *sk, *result;
struct hlist_node *node, *next;
struct hlist_nulls_node *node;
unsigned short hnum = ntohs(dport);
unsigned int hash = udp_hashfn(net, hnum);
struct udp_hslot *hslot = &udptable->hash[hash];
@ -108,19 +108,21 @@ static struct sock *__udp6_lib_lookup(struct net *net,
begin:
result = NULL;
badness = -1;
sk_for_each_rcu_safenext(sk, node, &hslot->head, next) {
/*
* lockless reader, and SLAB_DESTROY_BY_RCU items:
* We must check this item was not moved to another chain
*/
if (udp_hashfn(net, sk->sk_hash) != hash)
goto begin;
sk_nulls_for_each_rcu(sk, node, &hslot->head) {
score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif);
if (score > badness) {
result = sk;
badness = score;
}
}
/*
* if the nulls value we got at the end of this lookup is
* not the expected one, we must restart lookup.
* We probably met an item that was moved to another chain.
*/
if (get_nulls_value(node) != hash)
goto begin;
if (result) {
if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
result = NULL;
@ -374,11 +376,11 @@ static struct sock *udp_v6_mcast_next(struct net *net, struct sock *sk,
__be16 rmt_port, struct in6_addr *rmt_addr,
int dif)
{
struct hlist_node *node;
struct hlist_nulls_node *node;
struct sock *s = sk;
unsigned short num = ntohs(loc_port);
sk_for_each_from(s, node) {
sk_nulls_for_each_from(s, node) {
struct inet_sock *inet = inet_sk(s);
if (!net_eq(sock_net(s), net))
@ -423,7 +425,7 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
int dif;
spin_lock(&hslot->lock);
sk = sk_head(&hslot->head);
sk = sk_nulls_head(&hslot->head);
dif = inet6_iif(skb);
sk = udp_v6_mcast_next(net, sk, uh->dest, daddr, uh->source, saddr, dif);
if (!sk) {
@ -432,7 +434,7 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
}
sk2 = sk;
while ((sk2 = udp_v6_mcast_next(net, sk_next(sk2), uh->dest, daddr,
while ((sk2 = udp_v6_mcast_next(net, sk_nulls_next(sk2), uh->dest, daddr,
uh->source, saddr, dif))) {
struct sk_buff *buff = skb_clone(skb, GFP_ATOMIC);
if (buff) {

Write Preview
Loading…
Cancel
Save