inet6_hashtables.c 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295
  1. /*
  2. * INET An implementation of the TCP/IP protocol suite for the LINUX
  3. * operating system. INET is implemented using the BSD Socket
  4. * interface as the means of communication with the user level.
  5. *
  6. * Generic INET6 transport hashtables
  7. *
  8. * Authors: Lotsa people, from code originally in tcp, generalised here
  9. * by Arnaldo Carvalho de Melo <acme@mandriva.com>
  10. *
  11. * This program is free software; you can redistribute it and/or
  12. * modify it under the terms of the GNU General Public License
  13. * as published by the Free Software Foundation; either version
  14. * 2 of the License, or (at your option) any later version.
  15. */
  16. #include <linux/module.h>
  17. #include <linux/random.h>
  18. #include <net/inet_connection_sock.h>
  19. #include <net/inet_hashtables.h>
  20. #include <net/inet6_hashtables.h>
  21. #include <net/secure_seq.h>
  22. #include <net/ip.h>
  23. int __inet6_hash(struct sock *sk, struct inet_timewait_sock *tw)
  24. {
  25. struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
  26. int twrefcnt = 0;
  27. WARN_ON(!sk_unhashed(sk));
  28. if (sk->sk_state == TCP_LISTEN) {
  29. struct inet_listen_hashbucket *ilb;
  30. ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
  31. spin_lock(&ilb->lock);
  32. __sk_nulls_add_node_rcu(sk, &ilb->head);
  33. spin_unlock(&ilb->lock);
  34. } else {
  35. unsigned int hash;
  36. struct hlist_nulls_head *list;
  37. spinlock_t *lock;
  38. sk->sk_hash = hash = inet6_sk_ehashfn(sk);
  39. list = &inet_ehash_bucket(hashinfo, hash)->chain;
  40. lock = inet_ehash_lockp(hashinfo, hash);
  41. spin_lock(lock);
  42. __sk_nulls_add_node_rcu(sk, list);
  43. if (tw) {
  44. WARN_ON(sk->sk_hash != tw->tw_hash);
  45. twrefcnt = inet_twsk_unhash(tw);
  46. }
  47. spin_unlock(lock);
  48. }
  49. sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  50. return twrefcnt;
  51. }
  52. EXPORT_SYMBOL(__inet6_hash);
  53. /*
  54. * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
  55. * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM
  56. *
  57. * The sockhash lock must be held as a reader here.
  58. */
  59. struct sock *__inet6_lookup_established(struct net *net,
  60. struct inet_hashinfo *hashinfo,
  61. const struct in6_addr *saddr,
  62. const __be16 sport,
  63. const struct in6_addr *daddr,
  64. const u16 hnum,
  65. const int dif)
  66. {
  67. struct sock *sk;
  68. const struct hlist_nulls_node *node;
  69. const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
  70. /* Optimize here for direct hit, only listening connections can
  71. * have wildcards anyways.
  72. */
  73. unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
  74. unsigned int slot = hash & hashinfo->ehash_mask;
  75. struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
  76. rcu_read_lock();
  77. begin:
  78. sk_nulls_for_each_rcu(sk, node, &head->chain) {
  79. if (sk->sk_hash != hash)
  80. continue;
  81. if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif))
  82. continue;
  83. if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
  84. goto out;
  85. if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif))) {
  86. sock_gen_put(sk);
  87. goto begin;
  88. }
  89. goto found;
  90. }
  91. if (get_nulls_value(node) != slot)
  92. goto begin;
  93. out:
  94. sk = NULL;
  95. found:
  96. rcu_read_unlock();
  97. return sk;
  98. }
  99. EXPORT_SYMBOL(__inet6_lookup_established);
  100. static inline int compute_score(struct sock *sk, struct net *net,
  101. const unsigned short hnum,
  102. const struct in6_addr *daddr,
  103. const int dif)
  104. {
  105. int score = -1;
  106. if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum &&
  107. sk->sk_family == PF_INET6) {
  108. score = 1;
  109. if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {
  110. if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
  111. return -1;
  112. score++;
  113. }
  114. if (sk->sk_bound_dev_if) {
  115. if (sk->sk_bound_dev_if != dif)
  116. return -1;
  117. score++;
  118. }
  119. }
  120. return score;
  121. }
  122. struct sock *inet6_lookup_listener(struct net *net,
  123. struct inet_hashinfo *hashinfo, const struct in6_addr *saddr,
  124. const __be16 sport, const struct in6_addr *daddr,
  125. const unsigned short hnum, const int dif)
  126. {
  127. struct sock *sk;
  128. const struct hlist_nulls_node *node;
  129. struct sock *result;
  130. int score, hiscore, matches = 0, reuseport = 0;
  131. u32 phash = 0;
  132. unsigned int hash = inet_lhashfn(net, hnum);
  133. struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
  134. rcu_read_lock();
  135. begin:
  136. result = NULL;
  137. hiscore = 0;
  138. sk_nulls_for_each(sk, node, &ilb->head) {
  139. score = compute_score(sk, net, hnum, daddr, dif);
  140. if (score > hiscore) {
  141. hiscore = score;
  142. result = sk;
  143. reuseport = sk->sk_reuseport;
  144. if (reuseport) {
  145. phash = inet6_ehashfn(net, daddr, hnum,
  146. saddr, sport);
  147. matches = 1;
  148. }
  149. } else if (score == hiscore && reuseport) {
  150. matches++;
  151. if (((u64)phash * matches) >> 32 == 0)
  152. result = sk;
  153. phash = next_pseudo_random32(phash);
  154. }
  155. }
  156. /*
  157. * if the nulls value we got at the end of this lookup is
  158. * not the expected one, we must restart lookup.
  159. * We probably met an item that was moved to another chain.
  160. */
  161. if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE)
  162. goto begin;
  163. if (result) {
  164. if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
  165. result = NULL;
  166. else if (unlikely(compute_score(result, net, hnum, daddr,
  167. dif) < hiscore)) {
  168. sock_put(result);
  169. goto begin;
  170. }
  171. }
  172. rcu_read_unlock();
  173. return result;
  174. }
  175. EXPORT_SYMBOL_GPL(inet6_lookup_listener);
  176. struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,
  177. const struct in6_addr *saddr, const __be16 sport,
  178. const struct in6_addr *daddr, const __be16 dport,
  179. const int dif)
  180. {
  181. struct sock *sk;
  182. local_bh_disable();
  183. sk = __inet6_lookup(net, hashinfo, saddr, sport, daddr, ntohs(dport), dif);
  184. local_bh_enable();
  185. return sk;
  186. }
  187. EXPORT_SYMBOL_GPL(inet6_lookup);
  188. static int __inet6_check_established(struct inet_timewait_death_row *death_row,
  189. struct sock *sk, const __u16 lport,
  190. struct inet_timewait_sock **twp)
  191. {
  192. struct inet_hashinfo *hinfo = death_row->hashinfo;
  193. struct inet_sock *inet = inet_sk(sk);
  194. const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr;
  195. const struct in6_addr *saddr = &sk->sk_v6_daddr;
  196. const int dif = sk->sk_bound_dev_if;
  197. const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
  198. struct net *net = sock_net(sk);
  199. const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr,
  200. inet->inet_dport);
  201. struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
  202. spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
  203. struct sock *sk2;
  204. const struct hlist_nulls_node *node;
  205. struct inet_timewait_sock *tw = NULL;
  206. int twrefcnt = 0;
  207. spin_lock(lock);
  208. sk_nulls_for_each(sk2, node, &head->chain) {
  209. if (sk2->sk_hash != hash)
  210. continue;
  211. if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, dif))) {
  212. if (sk2->sk_state == TCP_TIME_WAIT) {
  213. tw = inet_twsk(sk2);
  214. if (twsk_unique(sk, sk2, twp))
  215. break;
  216. }
  217. goto not_unique;
  218. }
  219. }
  220. /* Must record num and sport now. Otherwise we will see
  221. * in hash table socket with a funny identity.
  222. */
  223. inet->inet_num = lport;
  224. inet->inet_sport = htons(lport);
  225. sk->sk_hash = hash;
  226. WARN_ON(!sk_unhashed(sk));
  227. __sk_nulls_add_node_rcu(sk, &head->chain);
  228. if (tw) {
  229. twrefcnt = inet_twsk_unhash(tw);
  230. NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED);
  231. }
  232. spin_unlock(lock);
  233. if (twrefcnt)
  234. inet_twsk_put(tw);
  235. sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  236. if (twp) {
  237. *twp = tw;
  238. } else if (tw) {
  239. /* Silly. Should hash-dance instead... */
  240. inet_twsk_deschedule(tw, death_row);
  241. inet_twsk_put(tw);
  242. }
  243. return 0;
  244. not_unique:
  245. spin_unlock(lock);
  246. return -EADDRNOTAVAIL;
  247. }
  248. static inline u32 inet6_sk_port_offset(const struct sock *sk)
  249. {
  250. const struct inet_sock *inet = inet_sk(sk);
  251. return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32,
  252. sk->sk_v6_daddr.s6_addr32,
  253. inet->inet_dport);
  254. }
  255. int inet6_hash_connect(struct inet_timewait_death_row *death_row,
  256. struct sock *sk)
  257. {
  258. return __inet_hash_connect(death_row, sk, inet6_sk_port_offset(sk),
  259. __inet6_check_established, __inet6_hash);
  260. }
  261. EXPORT_SYMBOL_GPL(inet6_hash_connect);