inet6_hashtables.c 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328
  1. /*
  2. * INET An implementation of the TCP/IP protocol suite for the LINUX
  3. * operating system. INET is implemented using the BSD Socket
  4. * interface as the means of communication with the user level.
  5. *
  6. * Generic INET6 transport hashtables
  7. *
  8. * Authors: Lotsa people, from code originally in tcp, generalised here
  9. * by Arnaldo Carvalho de Melo <acme@mandriva.com>
  10. *
  11. * This program is free software; you can redistribute it and/or
  12. * modify it under the terms of the GNU General Public License
  13. * as published by the Free Software Foundation; either version
  14. * 2 of the License, or (at your option) any later version.
  15. */
  16. #include <linux/module.h>
  17. #include <linux/random.h>
  18. #include <net/inet_connection_sock.h>
  19. #include <net/inet_hashtables.h>
  20. #include <net/inet6_hashtables.h>
  21. #include <net/secure_seq.h>
  22. #include <net/ip.h>
  23. static unsigned int inet6_ehashfn(struct net *net,
  24. const struct in6_addr *laddr,
  25. const u16 lport,
  26. const struct in6_addr *faddr,
  27. const __be16 fport)
  28. {
  29. static u32 inet6_ehash_secret __read_mostly;
  30. static u32 ipv6_hash_secret __read_mostly;
  31. u32 lhash, fhash;
  32. net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));
  33. net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret));
  34. lhash = (__force u32)laddr->s6_addr32[3];
  35. fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret);
  36. return __inet6_ehashfn(lhash, lport, fhash, fport,
  37. inet6_ehash_secret + net_hash_mix(net));
  38. }
  39. static int inet6_sk_ehashfn(const struct sock *sk)
  40. {
  41. const struct inet_sock *inet = inet_sk(sk);
  42. const struct in6_addr *laddr = &sk->sk_v6_rcv_saddr;
  43. const struct in6_addr *faddr = &sk->sk_v6_daddr;
  44. const __u16 lport = inet->inet_num;
  45. const __be16 fport = inet->inet_dport;
  46. struct net *net = sock_net(sk);
  47. return inet6_ehashfn(net, laddr, lport, faddr, fport);
  48. }
  49. int __inet6_hash(struct sock *sk, struct inet_timewait_sock *tw)
  50. {
  51. struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
  52. int twrefcnt = 0;
  53. WARN_ON(!sk_unhashed(sk));
  54. if (sk->sk_state == TCP_LISTEN) {
  55. struct inet_listen_hashbucket *ilb;
  56. ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
  57. spin_lock(&ilb->lock);
  58. __sk_nulls_add_node_rcu(sk, &ilb->head);
  59. spin_unlock(&ilb->lock);
  60. } else {
  61. unsigned int hash;
  62. struct hlist_nulls_head *list;
  63. spinlock_t *lock;
  64. sk->sk_hash = hash = inet6_sk_ehashfn(sk);
  65. list = &inet_ehash_bucket(hashinfo, hash)->chain;
  66. lock = inet_ehash_lockp(hashinfo, hash);
  67. spin_lock(lock);
  68. __sk_nulls_add_node_rcu(sk, list);
  69. if (tw) {
  70. WARN_ON(sk->sk_hash != tw->tw_hash);
  71. twrefcnt = inet_twsk_unhash(tw);
  72. }
  73. spin_unlock(lock);
  74. }
  75. sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  76. return twrefcnt;
  77. }
  78. EXPORT_SYMBOL(__inet6_hash);
  79. /*
  80. * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
  81. * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM
  82. *
  83. * The sockhash lock must be held as a reader here.
  84. */
  85. struct sock *__inet6_lookup_established(struct net *net,
  86. struct inet_hashinfo *hashinfo,
  87. const struct in6_addr *saddr,
  88. const __be16 sport,
  89. const struct in6_addr *daddr,
  90. const u16 hnum,
  91. const int dif)
  92. {
  93. struct sock *sk;
  94. const struct hlist_nulls_node *node;
  95. const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
  96. /* Optimize here for direct hit, only listening connections can
  97. * have wildcards anyways.
  98. */
  99. unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
  100. unsigned int slot = hash & hashinfo->ehash_mask;
  101. struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
  102. rcu_read_lock();
  103. begin:
  104. sk_nulls_for_each_rcu(sk, node, &head->chain) {
  105. if (sk->sk_hash != hash)
  106. continue;
  107. if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif))
  108. continue;
  109. if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
  110. goto out;
  111. if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif))) {
  112. sock_gen_put(sk);
  113. goto begin;
  114. }
  115. goto found;
  116. }
  117. if (get_nulls_value(node) != slot)
  118. goto begin;
  119. out:
  120. sk = NULL;
  121. found:
  122. rcu_read_unlock();
  123. return sk;
  124. }
  125. EXPORT_SYMBOL(__inet6_lookup_established);
  126. static inline int compute_score(struct sock *sk, struct net *net,
  127. const unsigned short hnum,
  128. const struct in6_addr *daddr,
  129. const int dif)
  130. {
  131. int score = -1;
  132. if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum &&
  133. sk->sk_family == PF_INET6) {
  134. score = 1;
  135. if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {
  136. if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
  137. return -1;
  138. score++;
  139. }
  140. if (sk->sk_bound_dev_if) {
  141. if (sk->sk_bound_dev_if != dif)
  142. return -1;
  143. score++;
  144. }
  145. }
  146. return score;
  147. }
  148. struct sock *inet6_lookup_listener(struct net *net,
  149. struct inet_hashinfo *hashinfo, const struct in6_addr *saddr,
  150. const __be16 sport, const struct in6_addr *daddr,
  151. const unsigned short hnum, const int dif)
  152. {
  153. struct sock *sk;
  154. const struct hlist_nulls_node *node;
  155. struct sock *result;
  156. int score, hiscore, matches = 0, reuseport = 0;
  157. u32 phash = 0;
  158. unsigned int hash = inet_lhashfn(net, hnum);
  159. struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
  160. rcu_read_lock();
  161. begin:
  162. result = NULL;
  163. hiscore = 0;
  164. sk_nulls_for_each(sk, node, &ilb->head) {
  165. score = compute_score(sk, net, hnum, daddr, dif);
  166. if (score > hiscore) {
  167. hiscore = score;
  168. result = sk;
  169. reuseport = sk->sk_reuseport;
  170. if (reuseport) {
  171. phash = inet6_ehashfn(net, daddr, hnum,
  172. saddr, sport);
  173. matches = 1;
  174. }
  175. } else if (score == hiscore && reuseport) {
  176. matches++;
  177. if (((u64)phash * matches) >> 32 == 0)
  178. result = sk;
  179. phash = next_pseudo_random32(phash);
  180. }
  181. }
  182. /*
  183. * if the nulls value we got at the end of this lookup is
  184. * not the expected one, we must restart lookup.
  185. * We probably met an item that was moved to another chain.
  186. */
  187. if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE)
  188. goto begin;
  189. if (result) {
  190. if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
  191. result = NULL;
  192. else if (unlikely(compute_score(result, net, hnum, daddr,
  193. dif) < hiscore)) {
  194. sock_put(result);
  195. goto begin;
  196. }
  197. }
  198. rcu_read_unlock();
  199. return result;
  200. }
  201. EXPORT_SYMBOL_GPL(inet6_lookup_listener);
  202. struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,
  203. const struct in6_addr *saddr, const __be16 sport,
  204. const struct in6_addr *daddr, const __be16 dport,
  205. const int dif)
  206. {
  207. struct sock *sk;
  208. local_bh_disable();
  209. sk = __inet6_lookup(net, hashinfo, saddr, sport, daddr, ntohs(dport), dif);
  210. local_bh_enable();
  211. return sk;
  212. }
  213. EXPORT_SYMBOL_GPL(inet6_lookup);
  214. static int __inet6_check_established(struct inet_timewait_death_row *death_row,
  215. struct sock *sk, const __u16 lport,
  216. struct inet_timewait_sock **twp)
  217. {
  218. struct inet_hashinfo *hinfo = death_row->hashinfo;
  219. struct inet_sock *inet = inet_sk(sk);
  220. const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr;
  221. const struct in6_addr *saddr = &sk->sk_v6_daddr;
  222. const int dif = sk->sk_bound_dev_if;
  223. const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
  224. struct net *net = sock_net(sk);
  225. const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr,
  226. inet->inet_dport);
  227. struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
  228. spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
  229. struct sock *sk2;
  230. const struct hlist_nulls_node *node;
  231. struct inet_timewait_sock *tw = NULL;
  232. int twrefcnt = 0;
  233. spin_lock(lock);
  234. sk_nulls_for_each(sk2, node, &head->chain) {
  235. if (sk2->sk_hash != hash)
  236. continue;
  237. if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, dif))) {
  238. if (sk2->sk_state == TCP_TIME_WAIT) {
  239. tw = inet_twsk(sk2);
  240. if (twsk_unique(sk, sk2, twp))
  241. break;
  242. }
  243. goto not_unique;
  244. }
  245. }
  246. /* Must record num and sport now. Otherwise we will see
  247. * in hash table socket with a funny identity.
  248. */
  249. inet->inet_num = lport;
  250. inet->inet_sport = htons(lport);
  251. sk->sk_hash = hash;
  252. WARN_ON(!sk_unhashed(sk));
  253. __sk_nulls_add_node_rcu(sk, &head->chain);
  254. if (tw) {
  255. twrefcnt = inet_twsk_unhash(tw);
  256. NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED);
  257. }
  258. spin_unlock(lock);
  259. if (twrefcnt)
  260. inet_twsk_put(tw);
  261. sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  262. if (twp) {
  263. *twp = tw;
  264. } else if (tw) {
  265. /* Silly. Should hash-dance instead... */
  266. inet_twsk_deschedule(tw, death_row);
  267. inet_twsk_put(tw);
  268. }
  269. return 0;
  270. not_unique:
  271. spin_unlock(lock);
  272. return -EADDRNOTAVAIL;
  273. }
  274. static inline u32 inet6_sk_port_offset(const struct sock *sk)
  275. {
  276. const struct inet_sock *inet = inet_sk(sk);
  277. return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32,
  278. sk->sk_v6_daddr.s6_addr32,
  279. inet->inet_dport);
  280. }
  281. int inet6_hash_connect(struct inet_timewait_death_row *death_row,
  282. struct sock *sk)
  283. {
  284. return __inet_hash_connect(death_row, sk, inet6_sk_port_offset(sk),
  285. __inet6_check_established, __inet6_hash);
  286. }
  287. EXPORT_SYMBOL_GPL(inet6_hash_connect);