tcp_metrics.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697
  1. #include <linux/rcupdate.h>
  2. #include <linux/spinlock.h>
  3. #include <linux/jiffies.h>
  4. #include <linux/bootmem.h>
  5. #include <linux/module.h>
  6. #include <linux/cache.h>
  7. #include <linux/slab.h>
  8. #include <linux/init.h>
  9. #include <linux/tcp.h>
  10. #include <net/inet_connection_sock.h>
  11. #include <net/net_namespace.h>
  12. #include <net/request_sock.h>
  13. #include <net/inetpeer.h>
  14. #include <net/sock.h>
  15. #include <net/ipv6.h>
  16. #include <net/dst.h>
  17. #include <net/tcp.h>
  18. int sysctl_tcp_nometrics_save __read_mostly;
  19. enum tcp_metric_index {
  20. TCP_METRIC_RTT,
  21. TCP_METRIC_RTTVAR,
  22. TCP_METRIC_SSTHRESH,
  23. TCP_METRIC_CWND,
  24. TCP_METRIC_REORDERING,
  25. /* Always last. */
  26. TCP_METRIC_MAX,
  27. };
  28. struct tcp_metrics_block {
  29. struct tcp_metrics_block __rcu *tcpm_next;
  30. struct inetpeer_addr tcpm_addr;
  31. unsigned long tcpm_stamp;
  32. u32 tcpm_ts;
  33. u32 tcpm_ts_stamp;
  34. u32 tcpm_lock;
  35. u32 tcpm_vals[TCP_METRIC_MAX];
  36. };
  37. static bool tcp_metric_locked(struct tcp_metrics_block *tm,
  38. enum tcp_metric_index idx)
  39. {
  40. return tm->tcpm_lock & (1 << idx);
  41. }
  42. static u32 tcp_metric_get(struct tcp_metrics_block *tm,
  43. enum tcp_metric_index idx)
  44. {
  45. return tm->tcpm_vals[idx];
  46. }
  47. static u32 tcp_metric_get_jiffies(struct tcp_metrics_block *tm,
  48. enum tcp_metric_index idx)
  49. {
  50. return msecs_to_jiffies(tm->tcpm_vals[idx]);
  51. }
  52. static void tcp_metric_set(struct tcp_metrics_block *tm,
  53. enum tcp_metric_index idx,
  54. u32 val)
  55. {
  56. tm->tcpm_vals[idx] = val;
  57. }
  58. static void tcp_metric_set_msecs(struct tcp_metrics_block *tm,
  59. enum tcp_metric_index idx,
  60. u32 val)
  61. {
  62. tm->tcpm_vals[idx] = jiffies_to_msecs(val);
  63. }
  64. static bool addr_same(const struct inetpeer_addr *a,
  65. const struct inetpeer_addr *b)
  66. {
  67. const struct in6_addr *a6, *b6;
  68. if (a->family != b->family)
  69. return false;
  70. if (a->family == AF_INET)
  71. return a->addr.a4 == b->addr.a4;
  72. a6 = (const struct in6_addr *) &a->addr.a6[0];
  73. b6 = (const struct in6_addr *) &b->addr.a6[0];
  74. return ipv6_addr_equal(a6, b6);
  75. }
  76. struct tcpm_hash_bucket {
  77. struct tcp_metrics_block __rcu *chain;
  78. };
  79. static DEFINE_SPINLOCK(tcp_metrics_lock);
  80. static void tcpm_suck_dst(struct tcp_metrics_block *tm, struct dst_entry *dst)
  81. {
  82. u32 val;
  83. val = 0;
  84. if (dst_metric_locked(dst, RTAX_RTT))
  85. val |= 1 << TCP_METRIC_RTT;
  86. if (dst_metric_locked(dst, RTAX_RTTVAR))
  87. val |= 1 << TCP_METRIC_RTTVAR;
  88. if (dst_metric_locked(dst, RTAX_SSTHRESH))
  89. val |= 1 << TCP_METRIC_SSTHRESH;
  90. if (dst_metric_locked(dst, RTAX_CWND))
  91. val |= 1 << TCP_METRIC_CWND;
  92. if (dst_metric_locked(dst, RTAX_REORDERING))
  93. val |= 1 << TCP_METRIC_REORDERING;
  94. tm->tcpm_lock = val;
  95. tm->tcpm_vals[TCP_METRIC_RTT] = dst_metric_raw(dst, RTAX_RTT);
  96. tm->tcpm_vals[TCP_METRIC_RTTVAR] = dst_metric_raw(dst, RTAX_RTTVAR);
  97. tm->tcpm_vals[TCP_METRIC_SSTHRESH] = dst_metric_raw(dst, RTAX_SSTHRESH);
  98. tm->tcpm_vals[TCP_METRIC_CWND] = dst_metric_raw(dst, RTAX_CWND);
  99. tm->tcpm_vals[TCP_METRIC_REORDERING] = dst_metric_raw(dst, RTAX_REORDERING);
  100. tm->tcpm_ts = 0;
  101. tm->tcpm_ts_stamp = 0;
  102. }
  103. static struct tcp_metrics_block *tcpm_new(struct dst_entry *dst,
  104. struct inetpeer_addr *addr,
  105. unsigned int hash,
  106. bool reclaim)
  107. {
  108. struct tcp_metrics_block *tm;
  109. struct net *net;
  110. spin_lock_bh(&tcp_metrics_lock);
  111. net = dev_net(dst->dev);
  112. if (unlikely(reclaim)) {
  113. struct tcp_metrics_block *oldest;
  114. oldest = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain);
  115. for (tm = rcu_dereference(oldest->tcpm_next); tm;
  116. tm = rcu_dereference(tm->tcpm_next)) {
  117. if (time_before(tm->tcpm_stamp, oldest->tcpm_stamp))
  118. oldest = tm;
  119. }
  120. tm = oldest;
  121. } else {
  122. tm = kmalloc(sizeof(*tm), GFP_ATOMIC);
  123. if (!tm)
  124. goto out_unlock;
  125. }
  126. tm->tcpm_addr = *addr;
  127. tm->tcpm_stamp = jiffies;
  128. tcpm_suck_dst(tm, dst);
  129. if (likely(!reclaim)) {
  130. tm->tcpm_next = net->ipv4.tcp_metrics_hash[hash].chain;
  131. rcu_assign_pointer(net->ipv4.tcp_metrics_hash[hash].chain, tm);
  132. }
  133. out_unlock:
  134. spin_unlock_bh(&tcp_metrics_lock);
  135. return tm;
  136. }
  137. #define TCP_METRICS_TIMEOUT (60 * 60 * HZ)
  138. static void tcpm_check_stamp(struct tcp_metrics_block *tm, struct dst_entry *dst)
  139. {
  140. if (tm && unlikely(time_after(jiffies, tm->tcpm_stamp + TCP_METRICS_TIMEOUT)))
  141. tcpm_suck_dst(tm, dst);
  142. }
  143. #define TCP_METRICS_RECLAIM_DEPTH 5
  144. #define TCP_METRICS_RECLAIM_PTR (struct tcp_metrics_block *) 0x1UL
  145. static struct tcp_metrics_block *tcp_get_encode(struct tcp_metrics_block *tm, int depth)
  146. {
  147. if (tm)
  148. return tm;
  149. if (depth > TCP_METRICS_RECLAIM_DEPTH)
  150. return TCP_METRICS_RECLAIM_PTR;
  151. return NULL;
  152. }
  153. static struct tcp_metrics_block *__tcp_get_metrics(const struct inetpeer_addr *addr,
  154. struct net *net, unsigned int hash)
  155. {
  156. struct tcp_metrics_block *tm;
  157. int depth = 0;
  158. for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
  159. tm = rcu_dereference(tm->tcpm_next)) {
  160. if (addr_same(&tm->tcpm_addr, addr))
  161. break;
  162. depth++;
  163. }
  164. return tcp_get_encode(tm, depth);
  165. }
  166. static struct tcp_metrics_block *__tcp_get_metrics_req(struct request_sock *req,
  167. struct dst_entry *dst)
  168. {
  169. struct tcp_metrics_block *tm;
  170. struct inetpeer_addr addr;
  171. unsigned int hash;
  172. struct net *net;
  173. addr.family = req->rsk_ops->family;
  174. switch (addr.family) {
  175. case AF_INET:
  176. addr.addr.a4 = inet_rsk(req)->rmt_addr;
  177. hash = (__force unsigned int) addr.addr.a4;
  178. break;
  179. case AF_INET6:
  180. *(struct in6_addr *)addr.addr.a6 = inet6_rsk(req)->rmt_addr;
  181. hash = ((__force unsigned int) addr.addr.a6[0] ^
  182. (__force unsigned int) addr.addr.a6[1] ^
  183. (__force unsigned int) addr.addr.a6[2] ^
  184. (__force unsigned int) addr.addr.a6[3]);
  185. break;
  186. default:
  187. return NULL;
  188. }
  189. hash ^= (hash >> 24) ^ (hash >> 16) ^ (hash >> 8);
  190. net = dev_net(dst->dev);
  191. hash &= net->ipv4.tcp_metrics_hash_mask;
  192. for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
  193. tm = rcu_dereference(tm->tcpm_next)) {
  194. if (addr_same(&tm->tcpm_addr, &addr))
  195. break;
  196. }
  197. tcpm_check_stamp(tm, dst);
  198. return tm;
  199. }
  200. static struct tcp_metrics_block *__tcp_get_metrics_tw(struct inet_timewait_sock *tw)
  201. {
  202. struct inet6_timewait_sock *tw6;
  203. struct tcp_metrics_block *tm;
  204. struct inetpeer_addr addr;
  205. unsigned int hash;
  206. struct net *net;
  207. addr.family = tw->tw_family;
  208. switch (addr.family) {
  209. case AF_INET:
  210. addr.addr.a4 = tw->tw_daddr;
  211. hash = (__force unsigned int) addr.addr.a4;
  212. break;
  213. case AF_INET6:
  214. tw6 = inet6_twsk((struct sock *)tw);
  215. *(struct in6_addr *)addr.addr.a6 = tw6->tw_v6_daddr;
  216. hash = ((__force unsigned int) addr.addr.a6[0] ^
  217. (__force unsigned int) addr.addr.a6[1] ^
  218. (__force unsigned int) addr.addr.a6[2] ^
  219. (__force unsigned int) addr.addr.a6[3]);
  220. break;
  221. default:
  222. return NULL;
  223. }
  224. hash ^= (hash >> 24) ^ (hash >> 16) ^ (hash >> 8);
  225. net = twsk_net(tw);
  226. hash &= net->ipv4.tcp_metrics_hash_mask;
  227. for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
  228. tm = rcu_dereference(tm->tcpm_next)) {
  229. if (addr_same(&tm->tcpm_addr, &addr))
  230. break;
  231. }
  232. return tm;
  233. }
  234. static struct tcp_metrics_block *tcp_get_metrics(struct sock *sk,
  235. struct dst_entry *dst,
  236. bool create)
  237. {
  238. struct tcp_metrics_block *tm;
  239. struct inetpeer_addr addr;
  240. unsigned int hash;
  241. struct net *net;
  242. bool reclaim;
  243. addr.family = sk->sk_family;
  244. switch (addr.family) {
  245. case AF_INET:
  246. addr.addr.a4 = inet_sk(sk)->inet_daddr;
  247. hash = (__force unsigned int) addr.addr.a4;
  248. break;
  249. case AF_INET6:
  250. *(struct in6_addr *)addr.addr.a6 = inet6_sk(sk)->daddr;
  251. hash = ((__force unsigned int) addr.addr.a6[0] ^
  252. (__force unsigned int) addr.addr.a6[1] ^
  253. (__force unsigned int) addr.addr.a6[2] ^
  254. (__force unsigned int) addr.addr.a6[3]);
  255. break;
  256. default:
  257. return NULL;
  258. }
  259. hash ^= (hash >> 24) ^ (hash >> 16) ^ (hash >> 8);
  260. net = dev_net(dst->dev);
  261. hash &= net->ipv4.tcp_metrics_hash_mask;
  262. tm = __tcp_get_metrics(&addr, net, hash);
  263. reclaim = false;
  264. if (tm == TCP_METRICS_RECLAIM_PTR) {
  265. reclaim = true;
  266. tm = NULL;
  267. }
  268. if (!tm && create)
  269. tm = tcpm_new(dst, &addr, hash, reclaim);
  270. else
  271. tcpm_check_stamp(tm, dst);
  272. return tm;
  273. }
  274. /* Save metrics learned by this TCP session. This function is called
  275. * only, when TCP finishes successfully i.e. when it enters TIME-WAIT
  276. * or goes from LAST-ACK to CLOSE.
  277. */
  278. void tcp_update_metrics(struct sock *sk)
  279. {
  280. const struct inet_connection_sock *icsk = inet_csk(sk);
  281. struct dst_entry *dst = __sk_dst_get(sk);
  282. struct tcp_sock *tp = tcp_sk(sk);
  283. struct tcp_metrics_block *tm;
  284. unsigned long rtt;
  285. u32 val;
  286. int m;
  287. if (sysctl_tcp_nometrics_save || !dst)
  288. return;
  289. if (dst->flags & DST_HOST)
  290. dst_confirm(dst);
  291. rcu_read_lock();
  292. if (icsk->icsk_backoff || !tp->srtt) {
  293. /* This session failed to estimate rtt. Why?
  294. * Probably, no packets returned in time. Reset our
  295. * results.
  296. */
  297. tm = tcp_get_metrics(sk, dst, false);
  298. if (tm && !tcp_metric_locked(tm, TCP_METRIC_RTT))
  299. tcp_metric_set(tm, TCP_METRIC_RTT, 0);
  300. goto out_unlock;
  301. } else
  302. tm = tcp_get_metrics(sk, dst, true);
  303. if (!tm)
  304. goto out_unlock;
  305. rtt = tcp_metric_get_jiffies(tm, TCP_METRIC_RTT);
  306. m = rtt - tp->srtt;
  307. /* If newly calculated rtt larger than stored one, store new
  308. * one. Otherwise, use EWMA. Remember, rtt overestimation is
  309. * always better than underestimation.
  310. */
  311. if (!tcp_metric_locked(tm, TCP_METRIC_RTT)) {
  312. if (m <= 0)
  313. rtt = tp->srtt;
  314. else
  315. rtt -= (m >> 3);
  316. tcp_metric_set_msecs(tm, TCP_METRIC_RTT, rtt);
  317. }
  318. if (!tcp_metric_locked(tm, TCP_METRIC_RTTVAR)) {
  319. unsigned long var;
  320. if (m < 0)
  321. m = -m;
  322. /* Scale deviation to rttvar fixed point */
  323. m >>= 1;
  324. if (m < tp->mdev)
  325. m = tp->mdev;
  326. var = tcp_metric_get_jiffies(tm, TCP_METRIC_RTTVAR);
  327. if (m >= var)
  328. var = m;
  329. else
  330. var -= (var - m) >> 2;
  331. tcp_metric_set_msecs(tm, TCP_METRIC_RTTVAR, var);
  332. }
  333. if (tcp_in_initial_slowstart(tp)) {
  334. /* Slow start still did not finish. */
  335. if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) {
  336. val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);
  337. if (val && (tp->snd_cwnd >> 1) > val)
  338. tcp_metric_set(tm, TCP_METRIC_SSTHRESH,
  339. tp->snd_cwnd >> 1);
  340. }
  341. if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {
  342. val = tcp_metric_get(tm, TCP_METRIC_CWND);
  343. if (tp->snd_cwnd > val)
  344. tcp_metric_set(tm, TCP_METRIC_CWND,
  345. tp->snd_cwnd);
  346. }
  347. } else if (tp->snd_cwnd > tp->snd_ssthresh &&
  348. icsk->icsk_ca_state == TCP_CA_Open) {
  349. /* Cong. avoidance phase, cwnd is reliable. */
  350. if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH))
  351. tcp_metric_set(tm, TCP_METRIC_SSTHRESH,
  352. max(tp->snd_cwnd >> 1, tp->snd_ssthresh));
  353. if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {
  354. val = tcp_metric_get(tm, TCP_METRIC_CWND);
  355. tcp_metric_set(tm, TCP_METRIC_CWND, (val + tp->snd_cwnd) >> 1);
  356. }
  357. } else {
  358. /* Else slow start did not finish, cwnd is non-sense,
  359. * ssthresh may be also invalid.
  360. */
  361. if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {
  362. val = tcp_metric_get(tm, TCP_METRIC_CWND);
  363. tcp_metric_set(tm, TCP_METRIC_CWND,
  364. (val + tp->snd_ssthresh) >> 1);
  365. }
  366. if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) {
  367. val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);
  368. if (val && tp->snd_ssthresh > val)
  369. tcp_metric_set(tm, TCP_METRIC_SSTHRESH,
  370. tp->snd_ssthresh);
  371. }
  372. if (!tcp_metric_locked(tm, TCP_METRIC_REORDERING)) {
  373. val = tcp_metric_get(tm, TCP_METRIC_REORDERING);
  374. if (val < tp->reordering &&
  375. tp->reordering != sysctl_tcp_reordering)
  376. tcp_metric_set(tm, TCP_METRIC_REORDERING,
  377. tp->reordering);
  378. }
  379. }
  380. tm->tcpm_stamp = jiffies;
  381. out_unlock:
  382. rcu_read_unlock();
  383. }
  384. /* Initialize metrics on socket. */
  385. void tcp_init_metrics(struct sock *sk)
  386. {
  387. struct dst_entry *dst = __sk_dst_get(sk);
  388. struct tcp_sock *tp = tcp_sk(sk);
  389. struct tcp_metrics_block *tm;
  390. u32 val;
  391. if (dst == NULL)
  392. goto reset;
  393. dst_confirm(dst);
  394. rcu_read_lock();
  395. tm = tcp_get_metrics(sk, dst, true);
  396. if (!tm) {
  397. rcu_read_unlock();
  398. goto reset;
  399. }
  400. if (tcp_metric_locked(tm, TCP_METRIC_CWND))
  401. tp->snd_cwnd_clamp = tcp_metric_get(tm, TCP_METRIC_CWND);
  402. val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);
  403. if (val) {
  404. tp->snd_ssthresh = val;
  405. if (tp->snd_ssthresh > tp->snd_cwnd_clamp)
  406. tp->snd_ssthresh = tp->snd_cwnd_clamp;
  407. } else {
  408. /* ssthresh may have been reduced unnecessarily during.
  409. * 3WHS. Restore it back to its initial default.
  410. */
  411. tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
  412. }
  413. val = tcp_metric_get(tm, TCP_METRIC_REORDERING);
  414. if (val && tp->reordering != val) {
  415. tcp_disable_fack(tp);
  416. tcp_disable_early_retrans(tp);
  417. tp->reordering = val;
  418. }
  419. val = tcp_metric_get(tm, TCP_METRIC_RTT);
  420. if (val == 0 || tp->srtt == 0) {
  421. rcu_read_unlock();
  422. goto reset;
  423. }
  424. /* Initial rtt is determined from SYN,SYN-ACK.
  425. * The segment is small and rtt may appear much
  426. * less than real one. Use per-dst memory
  427. * to make it more realistic.
  428. *
  429. * A bit of theory. RTT is time passed after "normal" sized packet
  430. * is sent until it is ACKed. In normal circumstances sending small
  431. * packets force peer to delay ACKs and calculation is correct too.
  432. * The algorithm is adaptive and, provided we follow specs, it
  433. * NEVER underestimate RTT. BUT! If peer tries to make some clever
  434. * tricks sort of "quick acks" for time long enough to decrease RTT
  435. * to low value, and then abruptly stops to do it and starts to delay
  436. * ACKs, wait for troubles.
  437. */
  438. val = msecs_to_jiffies(val);
  439. if (val > tp->srtt) {
  440. tp->srtt = val;
  441. tp->rtt_seq = tp->snd_nxt;
  442. }
  443. val = tcp_metric_get_jiffies(tm, TCP_METRIC_RTTVAR);
  444. if (val > tp->mdev) {
  445. tp->mdev = val;
  446. tp->mdev_max = tp->rttvar = max(tp->mdev, tcp_rto_min(sk));
  447. }
  448. rcu_read_unlock();
  449. tcp_set_rto(sk);
  450. reset:
  451. if (tp->srtt == 0) {
  452. /* RFC6298: 5.7 We've failed to get a valid RTT sample from
  453. * 3WHS. This is most likely due to retransmission,
  454. * including spurious one. Reset the RTO back to 3secs
  455. * from the more aggressive 1sec to avoid more spurious
  456. * retransmission.
  457. */
  458. tp->mdev = tp->mdev_max = tp->rttvar = TCP_TIMEOUT_FALLBACK;
  459. inet_csk(sk)->icsk_rto = TCP_TIMEOUT_FALLBACK;
  460. }
  461. /* Cut cwnd down to 1 per RFC5681 if SYN or SYN-ACK has been
  462. * retransmitted. In light of RFC6298 more aggressive 1sec
  463. * initRTO, we only reset cwnd when more than 1 SYN/SYN-ACK
  464. * retransmission has occurred.
  465. */
  466. if (tp->total_retrans > 1)
  467. tp->snd_cwnd = 1;
  468. else
  469. tp->snd_cwnd = tcp_init_cwnd(tp, dst);
  470. tp->snd_cwnd_stamp = tcp_time_stamp;
  471. }
  472. bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst, bool paws_check)
  473. {
  474. struct tcp_metrics_block *tm;
  475. bool ret;
  476. if (!dst)
  477. return false;
  478. rcu_read_lock();
  479. tm = __tcp_get_metrics_req(req, dst);
  480. if (paws_check) {
  481. if (tm &&
  482. (u32)get_seconds() - tm->tcpm_ts_stamp < TCP_PAWS_MSL &&
  483. (s32)(tm->tcpm_ts - req->ts_recent) > TCP_PAWS_WINDOW)
  484. ret = false;
  485. else
  486. ret = true;
  487. } else {
  488. if (tm && tcp_metric_get(tm, TCP_METRIC_RTT) && tm->tcpm_ts_stamp)
  489. ret = true;
  490. else
  491. ret = false;
  492. }
  493. rcu_read_unlock();
  494. return ret;
  495. }
  496. EXPORT_SYMBOL_GPL(tcp_peer_is_proven);
  497. void tcp_fetch_timewait_stamp(struct sock *sk, struct dst_entry *dst)
  498. {
  499. struct tcp_metrics_block *tm;
  500. rcu_read_lock();
  501. tm = tcp_get_metrics(sk, dst, true);
  502. if (tm) {
  503. struct tcp_sock *tp = tcp_sk(sk);
  504. if ((u32)get_seconds() - tm->tcpm_ts_stamp <= TCP_PAWS_MSL) {
  505. tp->rx_opt.ts_recent_stamp = tm->tcpm_ts_stamp;
  506. tp->rx_opt.ts_recent = tm->tcpm_ts;
  507. }
  508. }
  509. rcu_read_unlock();
  510. }
  511. EXPORT_SYMBOL_GPL(tcp_fetch_timewait_stamp);
  512. /* VJ's idea. Save last timestamp seen from this destination and hold
  513. * it at least for normal timewait interval to use for duplicate
  514. * segment detection in subsequent connections, before they enter
  515. * synchronized state.
  516. */
  517. bool tcp_remember_stamp(struct sock *sk)
  518. {
  519. struct dst_entry *dst = __sk_dst_get(sk);
  520. bool ret = false;
  521. if (dst) {
  522. struct tcp_metrics_block *tm;
  523. rcu_read_lock();
  524. tm = tcp_get_metrics(sk, dst, true);
  525. if (tm) {
  526. struct tcp_sock *tp = tcp_sk(sk);
  527. if ((s32)(tm->tcpm_ts - tp->rx_opt.ts_recent) <= 0 ||
  528. ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL &&
  529. tm->tcpm_ts_stamp <= (u32)tp->rx_opt.ts_recent_stamp)) {
  530. tm->tcpm_ts_stamp = (u32)tp->rx_opt.ts_recent_stamp;
  531. tm->tcpm_ts = tp->rx_opt.ts_recent;
  532. }
  533. ret = true;
  534. }
  535. rcu_read_unlock();
  536. }
  537. return ret;
  538. }
  539. bool tcp_tw_remember_stamp(struct inet_timewait_sock *tw)
  540. {
  541. struct tcp_metrics_block *tm;
  542. bool ret = false;
  543. rcu_read_lock();
  544. tm = __tcp_get_metrics_tw(tw);
  545. if (tw) {
  546. const struct tcp_timewait_sock *tcptw;
  547. struct sock *sk = (struct sock *) tw;
  548. tcptw = tcp_twsk(sk);
  549. if ((s32)(tm->tcpm_ts - tcptw->tw_ts_recent) <= 0 ||
  550. ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL &&
  551. tm->tcpm_ts_stamp <= (u32)tcptw->tw_ts_recent_stamp)) {
  552. tm->tcpm_ts_stamp = (u32)tcptw->tw_ts_recent_stamp;
  553. tm->tcpm_ts = tcptw->tw_ts_recent;
  554. }
  555. ret = true;
  556. }
  557. rcu_read_unlock();
  558. return ret;
  559. }
  560. static unsigned long tcpmhash_entries;
  561. static int __init set_tcpmhash_entries(char *str)
  562. {
  563. ssize_t ret;
  564. if (!str)
  565. return 0;
  566. ret = kstrtoul(str, 0, &tcpmhash_entries);
  567. if (ret)
  568. return 0;
  569. return 1;
  570. }
  571. __setup("tcpmhash_entries=", set_tcpmhash_entries);
  572. static int __net_init tcp_net_metrics_init(struct net *net)
  573. {
  574. int slots, size;
  575. slots = tcpmhash_entries;
  576. if (!slots) {
  577. if (totalram_pages >= 128 * 1024)
  578. slots = 16 * 1024;
  579. else
  580. slots = 8 * 1024;
  581. }
  582. size = slots * sizeof(struct tcpm_hash_bucket);
  583. net->ipv4.tcp_metrics_hash = kzalloc(size, GFP_KERNEL);
  584. if (!net->ipv4.tcp_metrics_hash)
  585. return -ENOMEM;
  586. net->ipv4.tcp_metrics_hash_mask = (slots - 1);
  587. return 0;
  588. }
  589. static void __net_exit tcp_net_metrics_exit(struct net *net)
  590. {
  591. kfree(net->ipv4.tcp_metrics_hash);
  592. }
  593. static __net_initdata struct pernet_operations tcp_net_metrics_ops = {
  594. .init = tcp_net_metrics_init,
  595. .exit = tcp_net_metrics_exit,
  596. };
  597. void __init tcp_metrics_init(void)
  598. {
  599. register_pernet_subsys(&tcp_net_metrics_ops);
  600. }