tcp_metrics.c 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092
  1. #include <linux/rcupdate.h>
  2. #include <linux/spinlock.h>
  3. #include <linux/jiffies.h>
  4. #include <linux/module.h>
  5. #include <linux/cache.h>
  6. #include <linux/slab.h>
  7. #include <linux/init.h>
  8. #include <linux/tcp.h>
  9. #include <linux/hash.h>
  10. #include <linux/tcp_metrics.h>
  11. #include <linux/vmalloc.h>
  12. #include <net/inet_connection_sock.h>
  13. #include <net/net_namespace.h>
  14. #include <net/request_sock.h>
  15. #include <net/inetpeer.h>
  16. #include <net/sock.h>
  17. #include <net/ipv6.h>
  18. #include <net/dst.h>
  19. #include <net/tcp.h>
  20. #include <net/genetlink.h>
  21. int sysctl_tcp_nometrics_save __read_mostly;
  22. struct tcp_fastopen_metrics {
  23. u16 mss;
  24. u16 syn_loss:10; /* Recurring Fast Open SYN losses */
  25. unsigned long last_syn_loss; /* Last Fast Open SYN loss */
  26. struct tcp_fastopen_cookie cookie;
  27. };
  28. struct tcp_metrics_block {
  29. struct tcp_metrics_block __rcu *tcpm_next;
  30. struct inetpeer_addr tcpm_addr;
  31. unsigned long tcpm_stamp;
  32. u32 tcpm_ts;
  33. u32 tcpm_ts_stamp;
  34. u32 tcpm_lock;
  35. u32 tcpm_vals[TCP_METRIC_MAX + 1];
  36. struct tcp_fastopen_metrics tcpm_fastopen;
  37. struct rcu_head rcu_head;
  38. };
  39. static bool tcp_metric_locked(struct tcp_metrics_block *tm,
  40. enum tcp_metric_index idx)
  41. {
  42. return tm->tcpm_lock & (1 << idx);
  43. }
  44. static u32 tcp_metric_get(struct tcp_metrics_block *tm,
  45. enum tcp_metric_index idx)
  46. {
  47. return tm->tcpm_vals[idx];
  48. }
  49. static u32 tcp_metric_get_jiffies(struct tcp_metrics_block *tm,
  50. enum tcp_metric_index idx)
  51. {
  52. return msecs_to_jiffies(tm->tcpm_vals[idx]);
  53. }
  54. static void tcp_metric_set(struct tcp_metrics_block *tm,
  55. enum tcp_metric_index idx,
  56. u32 val)
  57. {
  58. tm->tcpm_vals[idx] = val;
  59. }
  60. static void tcp_metric_set_msecs(struct tcp_metrics_block *tm,
  61. enum tcp_metric_index idx,
  62. u32 val)
  63. {
  64. tm->tcpm_vals[idx] = jiffies_to_msecs(val);
  65. }
  66. static bool addr_same(const struct inetpeer_addr *a,
  67. const struct inetpeer_addr *b)
  68. {
  69. const struct in6_addr *a6, *b6;
  70. if (a->family != b->family)
  71. return false;
  72. if (a->family == AF_INET)
  73. return a->addr.a4 == b->addr.a4;
  74. a6 = (const struct in6_addr *) &a->addr.a6[0];
  75. b6 = (const struct in6_addr *) &b->addr.a6[0];
  76. return ipv6_addr_equal(a6, b6);
  77. }
  78. struct tcpm_hash_bucket {
  79. struct tcp_metrics_block __rcu *chain;
  80. };
  81. static DEFINE_SPINLOCK(tcp_metrics_lock);
  82. static void tcpm_suck_dst(struct tcp_metrics_block *tm, struct dst_entry *dst,
  83. bool fastopen_clear)
  84. {
  85. u32 val;
  86. tm->tcpm_stamp = jiffies;
  87. val = 0;
  88. if (dst_metric_locked(dst, RTAX_RTT))
  89. val |= 1 << TCP_METRIC_RTT;
  90. if (dst_metric_locked(dst, RTAX_RTTVAR))
  91. val |= 1 << TCP_METRIC_RTTVAR;
  92. if (dst_metric_locked(dst, RTAX_SSTHRESH))
  93. val |= 1 << TCP_METRIC_SSTHRESH;
  94. if (dst_metric_locked(dst, RTAX_CWND))
  95. val |= 1 << TCP_METRIC_CWND;
  96. if (dst_metric_locked(dst, RTAX_REORDERING))
  97. val |= 1 << TCP_METRIC_REORDERING;
  98. tm->tcpm_lock = val;
  99. tm->tcpm_vals[TCP_METRIC_RTT] = dst_metric_raw(dst, RTAX_RTT);
  100. tm->tcpm_vals[TCP_METRIC_RTTVAR] = dst_metric_raw(dst, RTAX_RTTVAR);
  101. tm->tcpm_vals[TCP_METRIC_SSTHRESH] = dst_metric_raw(dst, RTAX_SSTHRESH);
  102. tm->tcpm_vals[TCP_METRIC_CWND] = dst_metric_raw(dst, RTAX_CWND);
  103. tm->tcpm_vals[TCP_METRIC_REORDERING] = dst_metric_raw(dst, RTAX_REORDERING);
  104. tm->tcpm_ts = 0;
  105. tm->tcpm_ts_stamp = 0;
  106. if (fastopen_clear) {
  107. tm->tcpm_fastopen.mss = 0;
  108. tm->tcpm_fastopen.syn_loss = 0;
  109. tm->tcpm_fastopen.cookie.len = 0;
  110. }
  111. }
  112. static struct tcp_metrics_block *tcpm_new(struct dst_entry *dst,
  113. struct inetpeer_addr *addr,
  114. unsigned int hash,
  115. bool reclaim)
  116. {
  117. struct tcp_metrics_block *tm;
  118. struct net *net;
  119. spin_lock_bh(&tcp_metrics_lock);
  120. net = dev_net(dst->dev);
  121. if (unlikely(reclaim)) {
  122. struct tcp_metrics_block *oldest;
  123. oldest = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain);
  124. for (tm = rcu_dereference(oldest->tcpm_next); tm;
  125. tm = rcu_dereference(tm->tcpm_next)) {
  126. if (time_before(tm->tcpm_stamp, oldest->tcpm_stamp))
  127. oldest = tm;
  128. }
  129. tm = oldest;
  130. } else {
  131. tm = kmalloc(sizeof(*tm), GFP_ATOMIC);
  132. if (!tm)
  133. goto out_unlock;
  134. }
  135. tm->tcpm_addr = *addr;
  136. tcpm_suck_dst(tm, dst, true);
  137. if (likely(!reclaim)) {
  138. tm->tcpm_next = net->ipv4.tcp_metrics_hash[hash].chain;
  139. rcu_assign_pointer(net->ipv4.tcp_metrics_hash[hash].chain, tm);
  140. }
  141. out_unlock:
  142. spin_unlock_bh(&tcp_metrics_lock);
  143. return tm;
  144. }
  145. #define TCP_METRICS_TIMEOUT (60 * 60 * HZ)
  146. static void tcpm_check_stamp(struct tcp_metrics_block *tm, struct dst_entry *dst)
  147. {
  148. if (tm && unlikely(time_after(jiffies, tm->tcpm_stamp + TCP_METRICS_TIMEOUT)))
  149. tcpm_suck_dst(tm, dst, false);
  150. }
  151. #define TCP_METRICS_RECLAIM_DEPTH 5
  152. #define TCP_METRICS_RECLAIM_PTR (struct tcp_metrics_block *) 0x1UL
  153. static struct tcp_metrics_block *tcp_get_encode(struct tcp_metrics_block *tm, int depth)
  154. {
  155. if (tm)
  156. return tm;
  157. if (depth > TCP_METRICS_RECLAIM_DEPTH)
  158. return TCP_METRICS_RECLAIM_PTR;
  159. return NULL;
  160. }
  161. static struct tcp_metrics_block *__tcp_get_metrics(const struct inetpeer_addr *addr,
  162. struct net *net, unsigned int hash)
  163. {
  164. struct tcp_metrics_block *tm;
  165. int depth = 0;
  166. for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
  167. tm = rcu_dereference(tm->tcpm_next)) {
  168. if (addr_same(&tm->tcpm_addr, addr))
  169. break;
  170. depth++;
  171. }
  172. return tcp_get_encode(tm, depth);
  173. }
  174. static struct tcp_metrics_block *__tcp_get_metrics_req(struct request_sock *req,
  175. struct dst_entry *dst)
  176. {
  177. struct tcp_metrics_block *tm;
  178. struct inetpeer_addr addr;
  179. unsigned int hash;
  180. struct net *net;
  181. addr.family = req->rsk_ops->family;
  182. switch (addr.family) {
  183. case AF_INET:
  184. addr.addr.a4 = inet_rsk(req)->ir_rmt_addr;
  185. hash = (__force unsigned int) addr.addr.a4;
  186. break;
  187. #if IS_ENABLED(CONFIG_IPV6)
  188. case AF_INET6:
  189. *(struct in6_addr *)addr.addr.a6 = inet_rsk(req)->ir_v6_rmt_addr;
  190. hash = ipv6_addr_hash(&inet_rsk(req)->ir_v6_rmt_addr);
  191. break;
  192. #endif
  193. default:
  194. return NULL;
  195. }
  196. net = dev_net(dst->dev);
  197. hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
  198. for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
  199. tm = rcu_dereference(tm->tcpm_next)) {
  200. if (addr_same(&tm->tcpm_addr, &addr))
  201. break;
  202. }
  203. tcpm_check_stamp(tm, dst);
  204. return tm;
  205. }
  206. static struct tcp_metrics_block *__tcp_get_metrics_tw(struct inet_timewait_sock *tw)
  207. {
  208. struct tcp_metrics_block *tm;
  209. struct inetpeer_addr addr;
  210. unsigned int hash;
  211. struct net *net;
  212. addr.family = tw->tw_family;
  213. switch (addr.family) {
  214. case AF_INET:
  215. addr.addr.a4 = tw->tw_daddr;
  216. hash = (__force unsigned int) addr.addr.a4;
  217. break;
  218. #if IS_ENABLED(CONFIG_IPV6)
  219. case AF_INET6:
  220. *(struct in6_addr *)addr.addr.a6 = tw->tw_v6_daddr;
  221. hash = ipv6_addr_hash(&tw->tw_v6_daddr);
  222. break;
  223. #endif
  224. default:
  225. return NULL;
  226. }
  227. net = twsk_net(tw);
  228. hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
  229. for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
  230. tm = rcu_dereference(tm->tcpm_next)) {
  231. if (addr_same(&tm->tcpm_addr, &addr))
  232. break;
  233. }
  234. return tm;
  235. }
  236. static struct tcp_metrics_block *tcp_get_metrics(struct sock *sk,
  237. struct dst_entry *dst,
  238. bool create)
  239. {
  240. struct tcp_metrics_block *tm;
  241. struct inetpeer_addr addr;
  242. unsigned int hash;
  243. struct net *net;
  244. bool reclaim;
  245. addr.family = sk->sk_family;
  246. switch (addr.family) {
  247. case AF_INET:
  248. addr.addr.a4 = inet_sk(sk)->inet_daddr;
  249. hash = (__force unsigned int) addr.addr.a4;
  250. break;
  251. #if IS_ENABLED(CONFIG_IPV6)
  252. case AF_INET6:
  253. *(struct in6_addr *)addr.addr.a6 = sk->sk_v6_daddr;
  254. hash = ipv6_addr_hash(&sk->sk_v6_daddr);
  255. break;
  256. #endif
  257. default:
  258. return NULL;
  259. }
  260. net = dev_net(dst->dev);
  261. hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
  262. tm = __tcp_get_metrics(&addr, net, hash);
  263. reclaim = false;
  264. if (tm == TCP_METRICS_RECLAIM_PTR) {
  265. reclaim = true;
  266. tm = NULL;
  267. }
  268. if (!tm && create)
  269. tm = tcpm_new(dst, &addr, hash, reclaim);
  270. else
  271. tcpm_check_stamp(tm, dst);
  272. return tm;
  273. }
  274. /* Save metrics learned by this TCP session. This function is called
  275. * only, when TCP finishes successfully i.e. when it enters TIME-WAIT
  276. * or goes from LAST-ACK to CLOSE.
  277. */
  278. void tcp_update_metrics(struct sock *sk)
  279. {
  280. const struct inet_connection_sock *icsk = inet_csk(sk);
  281. struct dst_entry *dst = __sk_dst_get(sk);
  282. struct tcp_sock *tp = tcp_sk(sk);
  283. struct tcp_metrics_block *tm;
  284. unsigned long rtt;
  285. u32 val;
  286. int m;
  287. if (sysctl_tcp_nometrics_save || !dst)
  288. return;
  289. if (dst->flags & DST_HOST)
  290. dst_confirm(dst);
  291. rcu_read_lock();
  292. if (icsk->icsk_backoff || !tp->srtt) {
  293. /* This session failed to estimate rtt. Why?
  294. * Probably, no packets returned in time. Reset our
  295. * results.
  296. */
  297. tm = tcp_get_metrics(sk, dst, false);
  298. if (tm && !tcp_metric_locked(tm, TCP_METRIC_RTT))
  299. tcp_metric_set(tm, TCP_METRIC_RTT, 0);
  300. goto out_unlock;
  301. } else
  302. tm = tcp_get_metrics(sk, dst, true);
  303. if (!tm)
  304. goto out_unlock;
  305. rtt = tcp_metric_get_jiffies(tm, TCP_METRIC_RTT);
  306. m = rtt - tp->srtt;
  307. /* If newly calculated rtt larger than stored one, store new
  308. * one. Otherwise, use EWMA. Remember, rtt overestimation is
  309. * always better than underestimation.
  310. */
  311. if (!tcp_metric_locked(tm, TCP_METRIC_RTT)) {
  312. if (m <= 0)
  313. rtt = tp->srtt;
  314. else
  315. rtt -= (m >> 3);
  316. tcp_metric_set_msecs(tm, TCP_METRIC_RTT, rtt);
  317. }
  318. if (!tcp_metric_locked(tm, TCP_METRIC_RTTVAR)) {
  319. unsigned long var;
  320. if (m < 0)
  321. m = -m;
  322. /* Scale deviation to rttvar fixed point */
  323. m >>= 1;
  324. if (m < tp->mdev)
  325. m = tp->mdev;
  326. var = tcp_metric_get_jiffies(tm, TCP_METRIC_RTTVAR);
  327. if (m >= var)
  328. var = m;
  329. else
  330. var -= (var - m) >> 2;
  331. tcp_metric_set_msecs(tm, TCP_METRIC_RTTVAR, var);
  332. }
  333. if (tcp_in_initial_slowstart(tp)) {
  334. /* Slow start still did not finish. */
  335. if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) {
  336. val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);
  337. if (val && (tp->snd_cwnd >> 1) > val)
  338. tcp_metric_set(tm, TCP_METRIC_SSTHRESH,
  339. tp->snd_cwnd >> 1);
  340. }
  341. if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {
  342. val = tcp_metric_get(tm, TCP_METRIC_CWND);
  343. if (tp->snd_cwnd > val)
  344. tcp_metric_set(tm, TCP_METRIC_CWND,
  345. tp->snd_cwnd);
  346. }
  347. } else if (tp->snd_cwnd > tp->snd_ssthresh &&
  348. icsk->icsk_ca_state == TCP_CA_Open) {
  349. /* Cong. avoidance phase, cwnd is reliable. */
  350. if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH))
  351. tcp_metric_set(tm, TCP_METRIC_SSTHRESH,
  352. max(tp->snd_cwnd >> 1, tp->snd_ssthresh));
  353. if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {
  354. val = tcp_metric_get(tm, TCP_METRIC_CWND);
  355. tcp_metric_set(tm, TCP_METRIC_CWND, (val + tp->snd_cwnd) >> 1);
  356. }
  357. } else {
  358. /* Else slow start did not finish, cwnd is non-sense,
  359. * ssthresh may be also invalid.
  360. */
  361. if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {
  362. val = tcp_metric_get(tm, TCP_METRIC_CWND);
  363. tcp_metric_set(tm, TCP_METRIC_CWND,
  364. (val + tp->snd_ssthresh) >> 1);
  365. }
  366. if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) {
  367. val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);
  368. if (val && tp->snd_ssthresh > val)
  369. tcp_metric_set(tm, TCP_METRIC_SSTHRESH,
  370. tp->snd_ssthresh);
  371. }
  372. if (!tcp_metric_locked(tm, TCP_METRIC_REORDERING)) {
  373. val = tcp_metric_get(tm, TCP_METRIC_REORDERING);
  374. if (val < tp->reordering &&
  375. tp->reordering != sysctl_tcp_reordering)
  376. tcp_metric_set(tm, TCP_METRIC_REORDERING,
  377. tp->reordering);
  378. }
  379. }
  380. tm->tcpm_stamp = jiffies;
  381. out_unlock:
  382. rcu_read_unlock();
  383. }
  384. /* Initialize metrics on socket. */
  385. void tcp_init_metrics(struct sock *sk)
  386. {
  387. struct dst_entry *dst = __sk_dst_get(sk);
  388. struct tcp_sock *tp = tcp_sk(sk);
  389. struct tcp_metrics_block *tm;
  390. u32 val, crtt = 0; /* cached RTT scaled by 8 */
  391. if (dst == NULL)
  392. goto reset;
  393. dst_confirm(dst);
  394. rcu_read_lock();
  395. tm = tcp_get_metrics(sk, dst, true);
  396. if (!tm) {
  397. rcu_read_unlock();
  398. goto reset;
  399. }
  400. if (tcp_metric_locked(tm, TCP_METRIC_CWND))
  401. tp->snd_cwnd_clamp = tcp_metric_get(tm, TCP_METRIC_CWND);
  402. val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);
  403. if (val) {
  404. tp->snd_ssthresh = val;
  405. if (tp->snd_ssthresh > tp->snd_cwnd_clamp)
  406. tp->snd_ssthresh = tp->snd_cwnd_clamp;
  407. } else {
  408. /* ssthresh may have been reduced unnecessarily during.
  409. * 3WHS. Restore it back to its initial default.
  410. */
  411. tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
  412. }
  413. val = tcp_metric_get(tm, TCP_METRIC_REORDERING);
  414. if (val && tp->reordering != val) {
  415. tcp_disable_fack(tp);
  416. tcp_disable_early_retrans(tp);
  417. tp->reordering = val;
  418. }
  419. crtt = tcp_metric_get_jiffies(tm, TCP_METRIC_RTT);
  420. rcu_read_unlock();
  421. reset:
  422. /* The initial RTT measurement from the SYN/SYN-ACK is not ideal
  423. * to seed the RTO for later data packets because SYN packets are
  424. * small. Use the per-dst cached values to seed the RTO but keep
  425. * the RTT estimator variables intact (e.g., srtt, mdev, rttvar).
  426. * Later the RTO will be updated immediately upon obtaining the first
  427. * data RTT sample (tcp_rtt_estimator()). Hence the cached RTT only
  428. * influences the first RTO but not later RTT estimation.
  429. *
  430. * But if RTT is not available from the SYN (due to retransmits or
  431. * syn cookies) or the cache, force a conservative 3secs timeout.
  432. *
  433. * A bit of theory. RTT is time passed after "normal" sized packet
  434. * is sent until it is ACKed. In normal circumstances sending small
  435. * packets force peer to delay ACKs and calculation is correct too.
  436. * The algorithm is adaptive and, provided we follow specs, it
  437. * NEVER underestimate RTT. BUT! If peer tries to make some clever
  438. * tricks sort of "quick acks" for time long enough to decrease RTT
  439. * to low value, and then abruptly stops to do it and starts to delay
  440. * ACKs, wait for troubles.
  441. */
  442. if (crtt > tp->srtt) {
  443. /* Set RTO like tcp_rtt_estimator(), but from cached RTT. */
  444. crtt >>= 3;
  445. inet_csk(sk)->icsk_rto = crtt + max(2 * crtt, tcp_rto_min(sk));
  446. } else if (tp->srtt == 0) {
  447. /* RFC6298: 5.7 We've failed to get a valid RTT sample from
  448. * 3WHS. This is most likely due to retransmission,
  449. * including spurious one. Reset the RTO back to 3secs
  450. * from the more aggressive 1sec to avoid more spurious
  451. * retransmission.
  452. */
  453. tp->mdev = tp->mdev_max = tp->rttvar = TCP_TIMEOUT_FALLBACK;
  454. inet_csk(sk)->icsk_rto = TCP_TIMEOUT_FALLBACK;
  455. }
  456. /* Cut cwnd down to 1 per RFC5681 if SYN or SYN-ACK has been
  457. * retransmitted. In light of RFC6298 more aggressive 1sec
  458. * initRTO, we only reset cwnd when more than 1 SYN/SYN-ACK
  459. * retransmission has occurred.
  460. */
  461. if (tp->total_retrans > 1)
  462. tp->snd_cwnd = 1;
  463. else
  464. tp->snd_cwnd = tcp_init_cwnd(tp, dst);
  465. tp->snd_cwnd_stamp = tcp_time_stamp;
  466. }
  467. bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst, bool paws_check)
  468. {
  469. struct tcp_metrics_block *tm;
  470. bool ret;
  471. if (!dst)
  472. return false;
  473. rcu_read_lock();
  474. tm = __tcp_get_metrics_req(req, dst);
  475. if (paws_check) {
  476. if (tm &&
  477. (u32)get_seconds() - tm->tcpm_ts_stamp < TCP_PAWS_MSL &&
  478. (s32)(tm->tcpm_ts - req->ts_recent) > TCP_PAWS_WINDOW)
  479. ret = false;
  480. else
  481. ret = true;
  482. } else {
  483. if (tm && tcp_metric_get(tm, TCP_METRIC_RTT) && tm->tcpm_ts_stamp)
  484. ret = true;
  485. else
  486. ret = false;
  487. }
  488. rcu_read_unlock();
  489. return ret;
  490. }
  491. EXPORT_SYMBOL_GPL(tcp_peer_is_proven);
  492. void tcp_fetch_timewait_stamp(struct sock *sk, struct dst_entry *dst)
  493. {
  494. struct tcp_metrics_block *tm;
  495. rcu_read_lock();
  496. tm = tcp_get_metrics(sk, dst, true);
  497. if (tm) {
  498. struct tcp_sock *tp = tcp_sk(sk);
  499. if ((u32)get_seconds() - tm->tcpm_ts_stamp <= TCP_PAWS_MSL) {
  500. tp->rx_opt.ts_recent_stamp = tm->tcpm_ts_stamp;
  501. tp->rx_opt.ts_recent = tm->tcpm_ts;
  502. }
  503. }
  504. rcu_read_unlock();
  505. }
  506. EXPORT_SYMBOL_GPL(tcp_fetch_timewait_stamp);
  507. /* VJ's idea. Save last timestamp seen from this destination and hold
  508. * it at least for normal timewait interval to use for duplicate
  509. * segment detection in subsequent connections, before they enter
  510. * synchronized state.
  511. */
  512. bool tcp_remember_stamp(struct sock *sk)
  513. {
  514. struct dst_entry *dst = __sk_dst_get(sk);
  515. bool ret = false;
  516. if (dst) {
  517. struct tcp_metrics_block *tm;
  518. rcu_read_lock();
  519. tm = tcp_get_metrics(sk, dst, true);
  520. if (tm) {
  521. struct tcp_sock *tp = tcp_sk(sk);
  522. if ((s32)(tm->tcpm_ts - tp->rx_opt.ts_recent) <= 0 ||
  523. ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL &&
  524. tm->tcpm_ts_stamp <= (u32)tp->rx_opt.ts_recent_stamp)) {
  525. tm->tcpm_ts_stamp = (u32)tp->rx_opt.ts_recent_stamp;
  526. tm->tcpm_ts = tp->rx_opt.ts_recent;
  527. }
  528. ret = true;
  529. }
  530. rcu_read_unlock();
  531. }
  532. return ret;
  533. }
  534. bool tcp_tw_remember_stamp(struct inet_timewait_sock *tw)
  535. {
  536. struct tcp_metrics_block *tm;
  537. bool ret = false;
  538. rcu_read_lock();
  539. tm = __tcp_get_metrics_tw(tw);
  540. if (tm) {
  541. const struct tcp_timewait_sock *tcptw;
  542. struct sock *sk = (struct sock *) tw;
  543. tcptw = tcp_twsk(sk);
  544. if ((s32)(tm->tcpm_ts - tcptw->tw_ts_recent) <= 0 ||
  545. ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL &&
  546. tm->tcpm_ts_stamp <= (u32)tcptw->tw_ts_recent_stamp)) {
  547. tm->tcpm_ts_stamp = (u32)tcptw->tw_ts_recent_stamp;
  548. tm->tcpm_ts = tcptw->tw_ts_recent;
  549. }
  550. ret = true;
  551. }
  552. rcu_read_unlock();
  553. return ret;
  554. }
  555. static DEFINE_SEQLOCK(fastopen_seqlock);
  556. void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
  557. struct tcp_fastopen_cookie *cookie,
  558. int *syn_loss, unsigned long *last_syn_loss)
  559. {
  560. struct tcp_metrics_block *tm;
  561. rcu_read_lock();
  562. tm = tcp_get_metrics(sk, __sk_dst_get(sk), false);
  563. if (tm) {
  564. struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen;
  565. unsigned int seq;
  566. do {
  567. seq = read_seqbegin(&fastopen_seqlock);
  568. if (tfom->mss)
  569. *mss = tfom->mss;
  570. *cookie = tfom->cookie;
  571. *syn_loss = tfom->syn_loss;
  572. *last_syn_loss = *syn_loss ? tfom->last_syn_loss : 0;
  573. } while (read_seqretry(&fastopen_seqlock, seq));
  574. }
  575. rcu_read_unlock();
  576. }
  577. void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
  578. struct tcp_fastopen_cookie *cookie, bool syn_lost)
  579. {
  580. struct tcp_metrics_block *tm;
  581. rcu_read_lock();
  582. tm = tcp_get_metrics(sk, __sk_dst_get(sk), true);
  583. if (tm) {
  584. struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen;
  585. write_seqlock_bh(&fastopen_seqlock);
  586. tfom->mss = mss;
  587. if (cookie->len > 0)
  588. tfom->cookie = *cookie;
  589. if (syn_lost) {
  590. ++tfom->syn_loss;
  591. tfom->last_syn_loss = jiffies;
  592. } else
  593. tfom->syn_loss = 0;
  594. write_sequnlock_bh(&fastopen_seqlock);
  595. }
  596. rcu_read_unlock();
  597. }
  598. static struct genl_family tcp_metrics_nl_family = {
  599. .id = GENL_ID_GENERATE,
  600. .hdrsize = 0,
  601. .name = TCP_METRICS_GENL_NAME,
  602. .version = TCP_METRICS_GENL_VERSION,
  603. .maxattr = TCP_METRICS_ATTR_MAX,
  604. .netnsok = true,
  605. };
  606. static struct nla_policy tcp_metrics_nl_policy[TCP_METRICS_ATTR_MAX + 1] = {
  607. [TCP_METRICS_ATTR_ADDR_IPV4] = { .type = NLA_U32, },
  608. [TCP_METRICS_ATTR_ADDR_IPV6] = { .type = NLA_BINARY,
  609. .len = sizeof(struct in6_addr), },
  610. /* Following attributes are not received for GET/DEL,
  611. * we keep them for reference
  612. */
  613. #if 0
  614. [TCP_METRICS_ATTR_AGE] = { .type = NLA_MSECS, },
  615. [TCP_METRICS_ATTR_TW_TSVAL] = { .type = NLA_U32, },
  616. [TCP_METRICS_ATTR_TW_TS_STAMP] = { .type = NLA_S32, },
  617. [TCP_METRICS_ATTR_VALS] = { .type = NLA_NESTED, },
  618. [TCP_METRICS_ATTR_FOPEN_MSS] = { .type = NLA_U16, },
  619. [TCP_METRICS_ATTR_FOPEN_SYN_DROPS] = { .type = NLA_U16, },
  620. [TCP_METRICS_ATTR_FOPEN_SYN_DROP_TS] = { .type = NLA_MSECS, },
  621. [TCP_METRICS_ATTR_FOPEN_COOKIE] = { .type = NLA_BINARY,
  622. .len = TCP_FASTOPEN_COOKIE_MAX, },
  623. #endif
  624. };
  625. /* Add attributes, caller cancels its header on failure */
  626. static int tcp_metrics_fill_info(struct sk_buff *msg,
  627. struct tcp_metrics_block *tm)
  628. {
  629. struct nlattr *nest;
  630. int i;
  631. switch (tm->tcpm_addr.family) {
  632. case AF_INET:
  633. if (nla_put_be32(msg, TCP_METRICS_ATTR_ADDR_IPV4,
  634. tm->tcpm_addr.addr.a4) < 0)
  635. goto nla_put_failure;
  636. break;
  637. case AF_INET6:
  638. if (nla_put(msg, TCP_METRICS_ATTR_ADDR_IPV6, 16,
  639. tm->tcpm_addr.addr.a6) < 0)
  640. goto nla_put_failure;
  641. break;
  642. default:
  643. return -EAFNOSUPPORT;
  644. }
  645. if (nla_put_msecs(msg, TCP_METRICS_ATTR_AGE,
  646. jiffies - tm->tcpm_stamp) < 0)
  647. goto nla_put_failure;
  648. if (tm->tcpm_ts_stamp) {
  649. if (nla_put_s32(msg, TCP_METRICS_ATTR_TW_TS_STAMP,
  650. (s32) (get_seconds() - tm->tcpm_ts_stamp)) < 0)
  651. goto nla_put_failure;
  652. if (nla_put_u32(msg, TCP_METRICS_ATTR_TW_TSVAL,
  653. tm->tcpm_ts) < 0)
  654. goto nla_put_failure;
  655. }
  656. {
  657. int n = 0;
  658. nest = nla_nest_start(msg, TCP_METRICS_ATTR_VALS);
  659. if (!nest)
  660. goto nla_put_failure;
  661. for (i = 0; i < TCP_METRIC_MAX + 1; i++) {
  662. if (!tm->tcpm_vals[i])
  663. continue;
  664. if (nla_put_u32(msg, i + 1, tm->tcpm_vals[i]) < 0)
  665. goto nla_put_failure;
  666. n++;
  667. }
  668. if (n)
  669. nla_nest_end(msg, nest);
  670. else
  671. nla_nest_cancel(msg, nest);
  672. }
  673. {
  674. struct tcp_fastopen_metrics tfom_copy[1], *tfom;
  675. unsigned int seq;
  676. do {
  677. seq = read_seqbegin(&fastopen_seqlock);
  678. tfom_copy[0] = tm->tcpm_fastopen;
  679. } while (read_seqretry(&fastopen_seqlock, seq));
  680. tfom = tfom_copy;
  681. if (tfom->mss &&
  682. nla_put_u16(msg, TCP_METRICS_ATTR_FOPEN_MSS,
  683. tfom->mss) < 0)
  684. goto nla_put_failure;
  685. if (tfom->syn_loss &&
  686. (nla_put_u16(msg, TCP_METRICS_ATTR_FOPEN_SYN_DROPS,
  687. tfom->syn_loss) < 0 ||
  688. nla_put_msecs(msg, TCP_METRICS_ATTR_FOPEN_SYN_DROP_TS,
  689. jiffies - tfom->last_syn_loss) < 0))
  690. goto nla_put_failure;
  691. if (tfom->cookie.len > 0 &&
  692. nla_put(msg, TCP_METRICS_ATTR_FOPEN_COOKIE,
  693. tfom->cookie.len, tfom->cookie.val) < 0)
  694. goto nla_put_failure;
  695. }
  696. return 0;
  697. nla_put_failure:
  698. return -EMSGSIZE;
  699. }
  700. static int tcp_metrics_dump_info(struct sk_buff *skb,
  701. struct netlink_callback *cb,
  702. struct tcp_metrics_block *tm)
  703. {
  704. void *hdr;
  705. hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
  706. &tcp_metrics_nl_family, NLM_F_MULTI,
  707. TCP_METRICS_CMD_GET);
  708. if (!hdr)
  709. return -EMSGSIZE;
  710. if (tcp_metrics_fill_info(skb, tm) < 0)
  711. goto nla_put_failure;
  712. return genlmsg_end(skb, hdr);
  713. nla_put_failure:
  714. genlmsg_cancel(skb, hdr);
  715. return -EMSGSIZE;
  716. }
  717. static int tcp_metrics_nl_dump(struct sk_buff *skb,
  718. struct netlink_callback *cb)
  719. {
  720. struct net *net = sock_net(skb->sk);
  721. unsigned int max_rows = 1U << net->ipv4.tcp_metrics_hash_log;
  722. unsigned int row, s_row = cb->args[0];
  723. int s_col = cb->args[1], col = s_col;
  724. for (row = s_row; row < max_rows; row++, s_col = 0) {
  725. struct tcp_metrics_block *tm;
  726. struct tcpm_hash_bucket *hb = net->ipv4.tcp_metrics_hash + row;
  727. rcu_read_lock();
  728. for (col = 0, tm = rcu_dereference(hb->chain); tm;
  729. tm = rcu_dereference(tm->tcpm_next), col++) {
  730. if (col < s_col)
  731. continue;
  732. if (tcp_metrics_dump_info(skb, cb, tm) < 0) {
  733. rcu_read_unlock();
  734. goto done;
  735. }
  736. }
  737. rcu_read_unlock();
  738. }
  739. done:
  740. cb->args[0] = row;
  741. cb->args[1] = col;
  742. return skb->len;
  743. }
  744. static int parse_nl_addr(struct genl_info *info, struct inetpeer_addr *addr,
  745. unsigned int *hash, int optional)
  746. {
  747. struct nlattr *a;
  748. a = info->attrs[TCP_METRICS_ATTR_ADDR_IPV4];
  749. if (a) {
  750. addr->family = AF_INET;
  751. addr->addr.a4 = nla_get_be32(a);
  752. *hash = (__force unsigned int) addr->addr.a4;
  753. return 0;
  754. }
  755. a = info->attrs[TCP_METRICS_ATTR_ADDR_IPV6];
  756. if (a) {
  757. if (nla_len(a) != sizeof(struct in6_addr))
  758. return -EINVAL;
  759. addr->family = AF_INET6;
  760. memcpy(addr->addr.a6, nla_data(a), sizeof(addr->addr.a6));
  761. *hash = ipv6_addr_hash((struct in6_addr *) addr->addr.a6);
  762. return 0;
  763. }
  764. return optional ? 1 : -EAFNOSUPPORT;
  765. }
  766. static int tcp_metrics_nl_cmd_get(struct sk_buff *skb, struct genl_info *info)
  767. {
  768. struct tcp_metrics_block *tm;
  769. struct inetpeer_addr addr;
  770. unsigned int hash;
  771. struct sk_buff *msg;
  772. struct net *net = genl_info_net(info);
  773. void *reply;
  774. int ret;
  775. ret = parse_nl_addr(info, &addr, &hash, 0);
  776. if (ret < 0)
  777. return ret;
  778. msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
  779. if (!msg)
  780. return -ENOMEM;
  781. reply = genlmsg_put_reply(msg, info, &tcp_metrics_nl_family, 0,
  782. info->genlhdr->cmd);
  783. if (!reply)
  784. goto nla_put_failure;
  785. hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
  786. ret = -ESRCH;
  787. rcu_read_lock();
  788. for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
  789. tm = rcu_dereference(tm->tcpm_next)) {
  790. if (addr_same(&tm->tcpm_addr, &addr)) {
  791. ret = tcp_metrics_fill_info(msg, tm);
  792. break;
  793. }
  794. }
  795. rcu_read_unlock();
  796. if (ret < 0)
  797. goto out_free;
  798. genlmsg_end(msg, reply);
  799. return genlmsg_reply(msg, info);
  800. nla_put_failure:
  801. ret = -EMSGSIZE;
  802. out_free:
  803. nlmsg_free(msg);
  804. return ret;
  805. }
  806. #define deref_locked_genl(p) \
  807. rcu_dereference_protected(p, lockdep_genl_is_held() && \
  808. lockdep_is_held(&tcp_metrics_lock))
  809. #define deref_genl(p) rcu_dereference_protected(p, lockdep_genl_is_held())
  810. static int tcp_metrics_flush_all(struct net *net)
  811. {
  812. unsigned int max_rows = 1U << net->ipv4.tcp_metrics_hash_log;
  813. struct tcpm_hash_bucket *hb = net->ipv4.tcp_metrics_hash;
  814. struct tcp_metrics_block *tm;
  815. unsigned int row;
  816. for (row = 0; row < max_rows; row++, hb++) {
  817. spin_lock_bh(&tcp_metrics_lock);
  818. tm = deref_locked_genl(hb->chain);
  819. if (tm)
  820. hb->chain = NULL;
  821. spin_unlock_bh(&tcp_metrics_lock);
  822. while (tm) {
  823. struct tcp_metrics_block *next;
  824. next = deref_genl(tm->tcpm_next);
  825. kfree_rcu(tm, rcu_head);
  826. tm = next;
  827. }
  828. }
  829. return 0;
  830. }
  831. static int tcp_metrics_nl_cmd_del(struct sk_buff *skb, struct genl_info *info)
  832. {
  833. struct tcpm_hash_bucket *hb;
  834. struct tcp_metrics_block *tm;
  835. struct tcp_metrics_block __rcu **pp;
  836. struct inetpeer_addr addr;
  837. unsigned int hash;
  838. struct net *net = genl_info_net(info);
  839. int ret;
  840. ret = parse_nl_addr(info, &addr, &hash, 1);
  841. if (ret < 0)
  842. return ret;
  843. if (ret > 0)
  844. return tcp_metrics_flush_all(net);
  845. hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
  846. hb = net->ipv4.tcp_metrics_hash + hash;
  847. pp = &hb->chain;
  848. spin_lock_bh(&tcp_metrics_lock);
  849. for (tm = deref_locked_genl(*pp); tm;
  850. pp = &tm->tcpm_next, tm = deref_locked_genl(*pp)) {
  851. if (addr_same(&tm->tcpm_addr, &addr)) {
  852. *pp = tm->tcpm_next;
  853. break;
  854. }
  855. }
  856. spin_unlock_bh(&tcp_metrics_lock);
  857. if (!tm)
  858. return -ESRCH;
  859. kfree_rcu(tm, rcu_head);
  860. return 0;
  861. }
  862. static struct genl_ops tcp_metrics_nl_ops[] = {
  863. {
  864. .cmd = TCP_METRICS_CMD_GET,
  865. .doit = tcp_metrics_nl_cmd_get,
  866. .dumpit = tcp_metrics_nl_dump,
  867. .policy = tcp_metrics_nl_policy,
  868. .flags = GENL_ADMIN_PERM,
  869. },
  870. {
  871. .cmd = TCP_METRICS_CMD_DEL,
  872. .doit = tcp_metrics_nl_cmd_del,
  873. .policy = tcp_metrics_nl_policy,
  874. .flags = GENL_ADMIN_PERM,
  875. },
  876. };
  877. static unsigned int tcpmhash_entries;
  878. static int __init set_tcpmhash_entries(char *str)
  879. {
  880. ssize_t ret;
  881. if (!str)
  882. return 0;
  883. ret = kstrtouint(str, 0, &tcpmhash_entries);
  884. if (ret)
  885. return 0;
  886. return 1;
  887. }
  888. __setup("tcpmhash_entries=", set_tcpmhash_entries);
  889. static int __net_init tcp_net_metrics_init(struct net *net)
  890. {
  891. size_t size;
  892. unsigned int slots;
  893. slots = tcpmhash_entries;
  894. if (!slots) {
  895. if (totalram_pages >= 128 * 1024)
  896. slots = 16 * 1024;
  897. else
  898. slots = 8 * 1024;
  899. }
  900. net->ipv4.tcp_metrics_hash_log = order_base_2(slots);
  901. size = sizeof(struct tcpm_hash_bucket) << net->ipv4.tcp_metrics_hash_log;
  902. net->ipv4.tcp_metrics_hash = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
  903. if (!net->ipv4.tcp_metrics_hash)
  904. net->ipv4.tcp_metrics_hash = vzalloc(size);
  905. if (!net->ipv4.tcp_metrics_hash)
  906. return -ENOMEM;
  907. return 0;
  908. }
  909. static void __net_exit tcp_net_metrics_exit(struct net *net)
  910. {
  911. unsigned int i;
  912. for (i = 0; i < (1U << net->ipv4.tcp_metrics_hash_log) ; i++) {
  913. struct tcp_metrics_block *tm, *next;
  914. tm = rcu_dereference_protected(net->ipv4.tcp_metrics_hash[i].chain, 1);
  915. while (tm) {
  916. next = rcu_dereference_protected(tm->tcpm_next, 1);
  917. kfree(tm);
  918. tm = next;
  919. }
  920. }
  921. if (is_vmalloc_addr(net->ipv4.tcp_metrics_hash))
  922. vfree(net->ipv4.tcp_metrics_hash);
  923. else
  924. kfree(net->ipv4.tcp_metrics_hash);
  925. }
  926. static __net_initdata struct pernet_operations tcp_net_metrics_ops = {
  927. .init = tcp_net_metrics_init,
  928. .exit = tcp_net_metrics_exit,
  929. };
  930. void __init tcp_metrics_init(void)
  931. {
  932. int ret;
  933. ret = register_pernet_subsys(&tcp_net_metrics_ops);
  934. if (ret < 0)
  935. goto cleanup;
  936. ret = genl_register_family_with_ops(&tcp_metrics_nl_family,
  937. tcp_metrics_nl_ops,
  938. ARRAY_SIZE(tcp_metrics_nl_ops));
  939. if (ret < 0)
  940. goto cleanup_subsys;
  941. return;
  942. cleanup_subsys:
  943. unregister_pernet_subsys(&tcp_net_metrics_ops);
  944. cleanup:
  945. return;
  946. }