sch_generic.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750
  1. /*
  2. * net/sched/sch_generic.c Generic packet scheduler routines.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public License
  6. * as published by the Free Software Foundation; either version
  7. * 2 of the License, or (at your option) any later version.
  8. *
  9. * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  10. * Jamal Hadi Salim, <hadi@cyberus.ca> 990601
  11. * - Ingress support
  12. */
  13. #include <linux/bitops.h>
  14. #include <linux/module.h>
  15. #include <linux/types.h>
  16. #include <linux/kernel.h>
  17. #include <linux/sched.h>
  18. #include <linux/string.h>
  19. #include <linux/errno.h>
  20. #include <linux/netdevice.h>
  21. #include <linux/skbuff.h>
  22. #include <linux/rtnetlink.h>
  23. #include <linux/init.h>
  24. #include <linux/rcupdate.h>
  25. #include <linux/list.h>
  26. #include <net/pkt_sched.h>
  27. /* Main transmission queue. */
  28. /* Modifications to data participating in scheduling must be protected with
  29. * qdisc_root_lock(qdisc) spinlock.
  30. *
  31. * The idea is the following:
  32. * - enqueue, dequeue are serialized via qdisc root lock
  33. * - ingress filtering is also serialized via qdisc root lock
  34. * - updates to tree and tree walking are only done under the rtnl mutex.
  35. */
  36. static inline int qdisc_qlen(struct Qdisc *q)
  37. {
  38. return q->q.qlen;
  39. }
  40. static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
  41. {
  42. if (unlikely(skb->next))
  43. q->gso_skb = skb;
  44. else
  45. q->ops->requeue(skb, q);
  46. __netif_schedule(q);
  47. return 0;
  48. }
  49. static inline struct sk_buff *dequeue_skb(struct Qdisc *q)
  50. {
  51. struct sk_buff *skb;
  52. if ((skb = q->gso_skb))
  53. q->gso_skb = NULL;
  54. else
  55. skb = q->dequeue(q);
  56. return skb;
  57. }
  58. static inline int handle_dev_cpu_collision(struct sk_buff *skb,
  59. struct netdev_queue *dev_queue,
  60. struct Qdisc *q)
  61. {
  62. int ret;
  63. if (unlikely(dev_queue->xmit_lock_owner == smp_processor_id())) {
  64. /*
  65. * Same CPU holding the lock. It may be a transient
  66. * configuration error, when hard_start_xmit() recurses. We
  67. * detect it by checking xmit owner and drop the packet when
  68. * deadloop is detected. Return OK to try the next skb.
  69. */
  70. kfree_skb(skb);
  71. if (net_ratelimit())
  72. printk(KERN_WARNING "Dead loop on netdevice %s, "
  73. "fix it urgently!\n", dev_queue->dev->name);
  74. ret = qdisc_qlen(q);
  75. } else {
  76. /*
  77. * Another cpu is holding lock, requeue & delay xmits for
  78. * some time.
  79. */
  80. __get_cpu_var(netdev_rx_stat).cpu_collision++;
  81. ret = dev_requeue_skb(skb, q);
  82. }
  83. return ret;
  84. }
  85. /*
  86. * NOTE: Called under qdisc_lock(q) with locally disabled BH.
  87. *
  88. * __QDISC_STATE_RUNNING guarantees only one CPU can process
  89. * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
  90. * this queue.
  91. *
  92. * netif_tx_lock serializes accesses to device driver.
  93. *
  94. * qdisc_lock(q) and netif_tx_lock are mutually exclusive,
  95. * if one is grabbed, another must be free.
  96. *
  97. * Note, that this procedure can be called by a watchdog timer
  98. *
  99. * Returns to the caller:
  100. * 0 - queue is empty or throttled.
  101. * >0 - queue is not empty.
  102. *
  103. */
  104. static inline int qdisc_restart(struct Qdisc *q)
  105. {
  106. struct netdev_queue *txq;
  107. int ret = NETDEV_TX_BUSY;
  108. struct net_device *dev;
  109. spinlock_t *root_lock;
  110. struct sk_buff *skb;
  111. /* Dequeue packet */
  112. if (unlikely((skb = dequeue_skb(q)) == NULL))
  113. return 0;
  114. root_lock = qdisc_root_lock(q);
  115. /* And release qdisc */
  116. spin_unlock(root_lock);
  117. dev = qdisc_dev(q);
  118. txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
  119. HARD_TX_LOCK(dev, txq, smp_processor_id());
  120. if (!netif_subqueue_stopped(dev, skb))
  121. ret = dev_hard_start_xmit(skb, dev, txq);
  122. HARD_TX_UNLOCK(dev, txq);
  123. spin_lock(root_lock);
  124. switch (ret) {
  125. case NETDEV_TX_OK:
  126. /* Driver sent out skb successfully */
  127. ret = qdisc_qlen(q);
  128. break;
  129. case NETDEV_TX_LOCKED:
  130. /* Driver try lock failed */
  131. ret = handle_dev_cpu_collision(skb, txq, q);
  132. break;
  133. default:
  134. /* Driver returned NETDEV_TX_BUSY - requeue skb */
  135. if (unlikely (ret != NETDEV_TX_BUSY && net_ratelimit()))
  136. printk(KERN_WARNING "BUG %s code %d qlen %d\n",
  137. dev->name, ret, q->q.qlen);
  138. ret = dev_requeue_skb(skb, q);
  139. break;
  140. }
  141. if (ret && netif_tx_queue_stopped(txq))
  142. ret = 0;
  143. return ret;
  144. }
  145. void __qdisc_run(struct Qdisc *q)
  146. {
  147. unsigned long start_time = jiffies;
  148. while (qdisc_restart(q)) {
  149. /*
  150. * Postpone processing if
  151. * 1. another process needs the CPU;
  152. * 2. we've been doing it for too long.
  153. */
  154. if (need_resched() || jiffies != start_time) {
  155. __netif_schedule(q);
  156. break;
  157. }
  158. }
  159. clear_bit(__QDISC_STATE_RUNNING, &q->state);
  160. }
  161. static void dev_watchdog(unsigned long arg)
  162. {
  163. struct net_device *dev = (struct net_device *)arg;
  164. netif_tx_lock(dev);
  165. if (!qdisc_tx_is_noop(dev)) {
  166. if (netif_device_present(dev) &&
  167. netif_running(dev) &&
  168. netif_carrier_ok(dev)) {
  169. int some_queue_stopped = 0;
  170. unsigned int i;
  171. for (i = 0; i < dev->num_tx_queues; i++) {
  172. struct netdev_queue *txq;
  173. txq = netdev_get_tx_queue(dev, i);
  174. if (netif_tx_queue_stopped(txq)) {
  175. some_queue_stopped = 1;
  176. break;
  177. }
  178. }
  179. if (some_queue_stopped &&
  180. time_after(jiffies, (dev->trans_start +
  181. dev->watchdog_timeo))) {
  182. char drivername[64];
  183. printk(KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit timed out\n",
  184. dev->name, netdev_drivername(dev, drivername, 64));
  185. dev->tx_timeout(dev);
  186. WARN_ON_ONCE(1);
  187. }
  188. if (!mod_timer(&dev->watchdog_timer,
  189. round_jiffies(jiffies +
  190. dev->watchdog_timeo)))
  191. dev_hold(dev);
  192. }
  193. }
  194. netif_tx_unlock(dev);
  195. dev_put(dev);
  196. }
  197. void __netdev_watchdog_up(struct net_device *dev)
  198. {
  199. if (dev->tx_timeout) {
  200. if (dev->watchdog_timeo <= 0)
  201. dev->watchdog_timeo = 5*HZ;
  202. if (!mod_timer(&dev->watchdog_timer,
  203. round_jiffies(jiffies + dev->watchdog_timeo)))
  204. dev_hold(dev);
  205. }
  206. }
  207. static void dev_watchdog_up(struct net_device *dev)
  208. {
  209. __netdev_watchdog_up(dev);
  210. }
  211. static void dev_watchdog_down(struct net_device *dev)
  212. {
  213. netif_tx_lock_bh(dev);
  214. if (del_timer(&dev->watchdog_timer))
  215. dev_put(dev);
  216. netif_tx_unlock_bh(dev);
  217. }
  218. /**
  219. * netif_carrier_on - set carrier
  220. * @dev: network device
  221. *
  222. * Device has detected that carrier.
  223. */
  224. void netif_carrier_on(struct net_device *dev)
  225. {
  226. if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
  227. linkwatch_fire_event(dev);
  228. if (netif_running(dev))
  229. __netdev_watchdog_up(dev);
  230. }
  231. }
  232. EXPORT_SYMBOL(netif_carrier_on);
  233. /**
  234. * netif_carrier_off - clear carrier
  235. * @dev: network device
  236. *
  237. * Device has detected loss of carrier.
  238. */
  239. void netif_carrier_off(struct net_device *dev)
  240. {
  241. if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state))
  242. linkwatch_fire_event(dev);
  243. }
  244. EXPORT_SYMBOL(netif_carrier_off);
  245. /* "NOOP" scheduler: the best scheduler, recommended for all interfaces
  246. under all circumstances. It is difficult to invent anything faster or
  247. cheaper.
  248. */
  249. static int noop_enqueue(struct sk_buff *skb, struct Qdisc * qdisc)
  250. {
  251. kfree_skb(skb);
  252. return NET_XMIT_CN;
  253. }
  254. static struct sk_buff *noop_dequeue(struct Qdisc * qdisc)
  255. {
  256. return NULL;
  257. }
  258. static int noop_requeue(struct sk_buff *skb, struct Qdisc* qdisc)
  259. {
  260. if (net_ratelimit())
  261. printk(KERN_DEBUG "%s deferred output. It is buggy.\n",
  262. skb->dev->name);
  263. kfree_skb(skb);
  264. return NET_XMIT_CN;
  265. }
  266. struct Qdisc_ops noop_qdisc_ops __read_mostly = {
  267. .id = "noop",
  268. .priv_size = 0,
  269. .enqueue = noop_enqueue,
  270. .dequeue = noop_dequeue,
  271. .requeue = noop_requeue,
  272. .owner = THIS_MODULE,
  273. };
  274. static struct netdev_queue noop_netdev_queue = {
  275. .qdisc = &noop_qdisc,
  276. };
  277. struct Qdisc noop_qdisc = {
  278. .enqueue = noop_enqueue,
  279. .dequeue = noop_dequeue,
  280. .flags = TCQ_F_BUILTIN,
  281. .ops = &noop_qdisc_ops,
  282. .list = LIST_HEAD_INIT(noop_qdisc.list),
  283. .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
  284. .dev_queue = &noop_netdev_queue,
  285. };
  286. EXPORT_SYMBOL(noop_qdisc);
  287. static struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
  288. .id = "noqueue",
  289. .priv_size = 0,
  290. .enqueue = noop_enqueue,
  291. .dequeue = noop_dequeue,
  292. .requeue = noop_requeue,
  293. .owner = THIS_MODULE,
  294. };
  295. static struct Qdisc noqueue_qdisc;
  296. static struct netdev_queue noqueue_netdev_queue = {
  297. .qdisc = &noqueue_qdisc,
  298. };
  299. static struct Qdisc noqueue_qdisc = {
  300. .enqueue = NULL,
  301. .dequeue = noop_dequeue,
  302. .flags = TCQ_F_BUILTIN,
  303. .ops = &noqueue_qdisc_ops,
  304. .list = LIST_HEAD_INIT(noqueue_qdisc.list),
  305. .q.lock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.q.lock),
  306. .dev_queue = &noqueue_netdev_queue,
  307. };
  308. static const u8 prio2band[TC_PRIO_MAX+1] =
  309. { 1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1 };
  310. /* 3-band FIFO queue: old style, but should be a bit faster than
  311. generic prio+fifo combination.
  312. */
  313. #define PFIFO_FAST_BANDS 3
  314. static inline struct sk_buff_head *prio2list(struct sk_buff *skb,
  315. struct Qdisc *qdisc)
  316. {
  317. struct sk_buff_head *list = qdisc_priv(qdisc);
  318. return list + prio2band[skb->priority & TC_PRIO_MAX];
  319. }
  320. static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc* qdisc)
  321. {
  322. struct sk_buff_head *list = prio2list(skb, qdisc);
  323. if (skb_queue_len(list) < qdisc_dev(qdisc)->tx_queue_len) {
  324. qdisc->q.qlen++;
  325. return __qdisc_enqueue_tail(skb, qdisc, list);
  326. }
  327. return qdisc_drop(skb, qdisc);
  328. }
  329. static struct sk_buff *pfifo_fast_dequeue(struct Qdisc* qdisc)
  330. {
  331. int prio;
  332. struct sk_buff_head *list = qdisc_priv(qdisc);
  333. for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
  334. if (!skb_queue_empty(list + prio)) {
  335. qdisc->q.qlen--;
  336. return __qdisc_dequeue_head(qdisc, list + prio);
  337. }
  338. }
  339. return NULL;
  340. }
  341. static int pfifo_fast_requeue(struct sk_buff *skb, struct Qdisc* qdisc)
  342. {
  343. qdisc->q.qlen++;
  344. return __qdisc_requeue(skb, qdisc, prio2list(skb, qdisc));
  345. }
  346. static void pfifo_fast_reset(struct Qdisc* qdisc)
  347. {
  348. int prio;
  349. struct sk_buff_head *list = qdisc_priv(qdisc);
  350. for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
  351. __qdisc_reset_queue(qdisc, list + prio);
  352. qdisc->qstats.backlog = 0;
  353. qdisc->q.qlen = 0;
  354. }
  355. static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
  356. {
  357. struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
  358. memcpy(&opt.priomap, prio2band, TC_PRIO_MAX+1);
  359. NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
  360. return skb->len;
  361. nla_put_failure:
  362. return -1;
  363. }
  364. static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt)
  365. {
  366. int prio;
  367. struct sk_buff_head *list = qdisc_priv(qdisc);
  368. for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
  369. skb_queue_head_init(list + prio);
  370. return 0;
  371. }
  372. static struct Qdisc_ops pfifo_fast_ops __read_mostly = {
  373. .id = "pfifo_fast",
  374. .priv_size = PFIFO_FAST_BANDS * sizeof(struct sk_buff_head),
  375. .enqueue = pfifo_fast_enqueue,
  376. .dequeue = pfifo_fast_dequeue,
  377. .requeue = pfifo_fast_requeue,
  378. .init = pfifo_fast_init,
  379. .reset = pfifo_fast_reset,
  380. .dump = pfifo_fast_dump,
  381. .owner = THIS_MODULE,
  382. };
  383. struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
  384. struct Qdisc_ops *ops)
  385. {
  386. void *p;
  387. struct Qdisc *sch;
  388. unsigned int size;
  389. int err = -ENOBUFS;
  390. /* ensure that the Qdisc and the private data are 32-byte aligned */
  391. size = QDISC_ALIGN(sizeof(*sch));
  392. size += ops->priv_size + (QDISC_ALIGNTO - 1);
  393. p = kzalloc(size, GFP_KERNEL);
  394. if (!p)
  395. goto errout;
  396. sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
  397. sch->padded = (char *) sch - (char *) p;
  398. INIT_LIST_HEAD(&sch->list);
  399. skb_queue_head_init(&sch->q);
  400. sch->ops = ops;
  401. sch->enqueue = ops->enqueue;
  402. sch->dequeue = ops->dequeue;
  403. sch->dev_queue = dev_queue;
  404. dev_hold(qdisc_dev(sch));
  405. atomic_set(&sch->refcnt, 1);
  406. return sch;
  407. errout:
  408. return ERR_PTR(err);
  409. }
  410. struct Qdisc * qdisc_create_dflt(struct net_device *dev,
  411. struct netdev_queue *dev_queue,
  412. struct Qdisc_ops *ops,
  413. unsigned int parentid)
  414. {
  415. struct Qdisc *sch;
  416. sch = qdisc_alloc(dev_queue, ops);
  417. if (IS_ERR(sch))
  418. goto errout;
  419. sch->parent = parentid;
  420. if (!ops->init || ops->init(sch, NULL) == 0)
  421. return sch;
  422. qdisc_destroy(sch);
  423. errout:
  424. return NULL;
  425. }
  426. EXPORT_SYMBOL(qdisc_create_dflt);
  427. /* Under qdisc_root_lock(qdisc) and BH! */
  428. void qdisc_reset(struct Qdisc *qdisc)
  429. {
  430. const struct Qdisc_ops *ops = qdisc->ops;
  431. if (ops->reset)
  432. ops->reset(qdisc);
  433. }
  434. EXPORT_SYMBOL(qdisc_reset);
  435. /* this is the rcu callback function to clean up a qdisc when there
  436. * are no further references to it */
  437. static void __qdisc_destroy(struct rcu_head *head)
  438. {
  439. struct Qdisc *qdisc = container_of(head, struct Qdisc, q_rcu);
  440. const struct Qdisc_ops *ops = qdisc->ops;
  441. #ifdef CONFIG_NET_SCHED
  442. qdisc_put_stab(qdisc->stab);
  443. #endif
  444. gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est);
  445. if (ops->reset)
  446. ops->reset(qdisc);
  447. if (ops->destroy)
  448. ops->destroy(qdisc);
  449. module_put(ops->owner);
  450. dev_put(qdisc_dev(qdisc));
  451. kfree_skb(qdisc->gso_skb);
  452. kfree((char *) qdisc - qdisc->padded);
  453. }
  454. /* Under qdisc_root_lock(qdisc) and BH! */
  455. void qdisc_destroy(struct Qdisc *qdisc)
  456. {
  457. if (qdisc->flags & TCQ_F_BUILTIN ||
  458. !atomic_dec_and_test(&qdisc->refcnt))
  459. return;
  460. if (qdisc->parent)
  461. list_del(&qdisc->list);
  462. call_rcu(&qdisc->q_rcu, __qdisc_destroy);
  463. }
  464. EXPORT_SYMBOL(qdisc_destroy);
  465. static bool dev_all_qdisc_sleeping_noop(struct net_device *dev)
  466. {
  467. unsigned int i;
  468. for (i = 0; i < dev->num_tx_queues; i++) {
  469. struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
  470. if (txq->qdisc_sleeping != &noop_qdisc)
  471. return false;
  472. }
  473. return true;
  474. }
  475. static void attach_one_default_qdisc(struct net_device *dev,
  476. struct netdev_queue *dev_queue,
  477. void *_unused)
  478. {
  479. struct Qdisc *qdisc;
  480. if (dev->tx_queue_len) {
  481. qdisc = qdisc_create_dflt(dev, dev_queue,
  482. &pfifo_fast_ops, TC_H_ROOT);
  483. if (!qdisc) {
  484. printk(KERN_INFO "%s: activation failed\n", dev->name);
  485. return;
  486. }
  487. } else {
  488. qdisc = &noqueue_qdisc;
  489. }
  490. dev_queue->qdisc_sleeping = qdisc;
  491. }
  492. static void transition_one_qdisc(struct net_device *dev,
  493. struct netdev_queue *dev_queue,
  494. void *_need_watchdog)
  495. {
  496. struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping;
  497. int *need_watchdog_p = _need_watchdog;
  498. rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
  499. if (new_qdisc != &noqueue_qdisc)
  500. *need_watchdog_p = 1;
  501. }
  502. void dev_activate(struct net_device *dev)
  503. {
  504. int need_watchdog;
  505. /* No queueing discipline is attached to device;
  506. create default one i.e. pfifo_fast for devices,
  507. which need queueing and noqueue_qdisc for
  508. virtual interfaces
  509. */
  510. if (dev_all_qdisc_sleeping_noop(dev))
  511. netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
  512. if (!netif_carrier_ok(dev))
  513. /* Delay activation until next carrier-on event */
  514. return;
  515. need_watchdog = 0;
  516. netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
  517. if (need_watchdog) {
  518. dev->trans_start = jiffies;
  519. dev_watchdog_up(dev);
  520. }
  521. }
  522. static void dev_deactivate_queue(struct net_device *dev,
  523. struct netdev_queue *dev_queue,
  524. void *_qdisc_default)
  525. {
  526. struct Qdisc *qdisc_default = _qdisc_default;
  527. struct Qdisc *qdisc;
  528. qdisc = dev_queue->qdisc;
  529. if (qdisc) {
  530. spin_lock_bh(qdisc_lock(qdisc));
  531. dev_queue->qdisc = qdisc_default;
  532. qdisc_reset(qdisc);
  533. spin_unlock_bh(qdisc_lock(qdisc));
  534. }
  535. }
  536. static bool some_qdisc_is_running(struct net_device *dev, int lock)
  537. {
  538. unsigned int i;
  539. for (i = 0; i < dev->num_tx_queues; i++) {
  540. struct netdev_queue *dev_queue;
  541. spinlock_t *root_lock;
  542. struct Qdisc *q;
  543. int val;
  544. dev_queue = netdev_get_tx_queue(dev, i);
  545. q = dev_queue->qdisc;
  546. root_lock = qdisc_root_lock(q);
  547. if (lock)
  548. spin_lock_bh(root_lock);
  549. val = test_bit(__QDISC_STATE_RUNNING, &q->state);
  550. if (lock)
  551. spin_unlock_bh(root_lock);
  552. if (val)
  553. return true;
  554. }
  555. return false;
  556. }
  557. void dev_deactivate(struct net_device *dev)
  558. {
  559. bool running;
  560. netdev_for_each_tx_queue(dev, dev_deactivate_queue, &noop_qdisc);
  561. dev_watchdog_down(dev);
  562. /* Wait for outstanding qdisc-less dev_queue_xmit calls. */
  563. synchronize_rcu();
  564. /* Wait for outstanding qdisc_run calls. */
  565. do {
  566. while (some_qdisc_is_running(dev, 0))
  567. yield();
  568. /*
  569. * Double-check inside queue lock to ensure that all effects
  570. * of the queue run are visible when we return.
  571. */
  572. running = some_qdisc_is_running(dev, 1);
  573. /*
  574. * The running flag should never be set at this point because
  575. * we've already set dev->qdisc to noop_qdisc *inside* the same
  576. * pair of spin locks. That is, if any qdisc_run starts after
  577. * our initial test it should see the noop_qdisc and then
  578. * clear the RUNNING bit before dropping the queue lock. So
  579. * if it is set here then we've found a bug.
  580. */
  581. } while (WARN_ON_ONCE(running));
  582. }
  583. static void dev_init_scheduler_queue(struct net_device *dev,
  584. struct netdev_queue *dev_queue,
  585. void *_qdisc)
  586. {
  587. struct Qdisc *qdisc = _qdisc;
  588. dev_queue->qdisc = qdisc;
  589. dev_queue->qdisc_sleeping = qdisc;
  590. }
  591. void dev_init_scheduler(struct net_device *dev)
  592. {
  593. netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
  594. dev_init_scheduler_queue(dev, &dev->rx_queue, NULL);
  595. setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev);
  596. }
  597. static void shutdown_scheduler_queue(struct net_device *dev,
  598. struct netdev_queue *dev_queue,
  599. void *_qdisc_default)
  600. {
  601. struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
  602. struct Qdisc *qdisc_default = _qdisc_default;
  603. if (qdisc) {
  604. spinlock_t *root_lock = qdisc_root_lock(qdisc);
  605. dev_queue->qdisc = qdisc_default;
  606. dev_queue->qdisc_sleeping = qdisc_default;
  607. spin_lock_bh(root_lock);
  608. qdisc_destroy(qdisc);
  609. spin_unlock_bh(root_lock);
  610. }
  611. }
  612. void dev_shutdown(struct net_device *dev)
  613. {
  614. netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
  615. shutdown_scheduler_queue(dev, &dev->rx_queue, NULL);
  616. WARN_ON(timer_pending(&dev->watchdog_timer));
  617. }