tick-broadcast.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802
  1. /*
  2. * linux/kernel/time/tick-broadcast.c
  3. *
  4. * This file contains functions which emulate a local clock-event
  5. * device via a broadcast event source.
  6. *
  7. * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
  8. * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
  9. * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
  10. *
  11. * This code is licenced under the GPL version 2. For details see
  12. * kernel-base/COPYING.
  13. */
  14. #include <linux/cpu.h>
  15. #include <linux/err.h>
  16. #include <linux/hrtimer.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/percpu.h>
  19. #include <linux/profile.h>
  20. #include <linux/sched.h>
  21. #include <linux/smp.h>
  22. #include "tick-internal.h"
  23. /*
  24. * Broadcast support for broken x86 hardware, where the local apic
  25. * timer stops in C3 state.
  26. */
  27. static struct tick_device tick_broadcast_device;
  28. static cpumask_var_t tick_broadcast_mask;
  29. static cpumask_var_t tmpmask;
  30. static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
  31. static int tick_broadcast_force;
  32. #ifdef CONFIG_TICK_ONESHOT
  33. static void tick_broadcast_clear_oneshot(int cpu);
  34. #else
  35. static inline void tick_broadcast_clear_oneshot(int cpu) { }
  36. #endif
  37. /*
  38. * Debugging: see timer_list.c
  39. */
  40. struct tick_device *tick_get_broadcast_device(void)
  41. {
  42. return &tick_broadcast_device;
  43. }
  44. struct cpumask *tick_get_broadcast_mask(void)
  45. {
  46. return tick_broadcast_mask;
  47. }
  48. /*
  49. * Start the device in periodic mode
  50. */
  51. static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  52. {
  53. if (bc)
  54. tick_setup_periodic(bc, 1);
  55. }
  56. /*
  57. * Check, if the device can be utilized as broadcast device:
  58. */
  59. int tick_check_broadcast_device(struct clock_event_device *dev)
  60. {
  61. struct clock_event_device *cur = tick_broadcast_device.evtdev;
  62. if ((dev->features & CLOCK_EVT_FEAT_DUMMY) ||
  63. (tick_broadcast_device.evtdev &&
  64. tick_broadcast_device.evtdev->rating >= dev->rating) ||
  65. (dev->features & CLOCK_EVT_FEAT_C3STOP))
  66. return 0;
  67. clockevents_exchange_device(tick_broadcast_device.evtdev, dev);
  68. if (cur)
  69. cur->event_handler = clockevents_handle_noop;
  70. tick_broadcast_device.evtdev = dev;
  71. if (!cpumask_empty(tick_broadcast_mask))
  72. tick_broadcast_start_periodic(dev);
  73. /*
  74. * Inform all cpus about this. We might be in a situation
  75. * where we did not switch to oneshot mode because the per cpu
  76. * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
  77. * of a oneshot capable broadcast device. Without that
  78. * notification the systems stays stuck in periodic mode
  79. * forever.
  80. */
  81. if (dev->features & CLOCK_EVT_FEAT_ONESHOT)
  82. tick_clock_notify();
  83. return 1;
  84. }
  85. /*
  86. * Check, if the device is the broadcast device
  87. */
  88. int tick_is_broadcast_device(struct clock_event_device *dev)
  89. {
  90. return (dev && tick_broadcast_device.evtdev == dev);
  91. }
  92. static void err_broadcast(const struct cpumask *mask)
  93. {
  94. pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
  95. }
  96. static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
  97. {
  98. if (!dev->broadcast)
  99. dev->broadcast = tick_broadcast;
  100. if (!dev->broadcast) {
  101. pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
  102. dev->name);
  103. dev->broadcast = err_broadcast;
  104. }
  105. }
  106. /*
  107. * Check, if the device is disfunctional and a place holder, which
  108. * needs to be handled by the broadcast device.
  109. */
  110. int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
  111. {
  112. unsigned long flags;
  113. int ret = 0;
  114. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  115. /*
  116. * Devices might be registered with both periodic and oneshot
  117. * mode disabled. This signals, that the device needs to be
  118. * operated from the broadcast device and is a placeholder for
  119. * the cpu local device.
  120. */
  121. if (!tick_device_is_functional(dev)) {
  122. dev->event_handler = tick_handle_periodic;
  123. tick_device_setup_broadcast_func(dev);
  124. cpumask_set_cpu(cpu, tick_broadcast_mask);
  125. tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
  126. ret = 1;
  127. } else {
  128. /*
  129. * When the new device is not affected by the stop
  130. * feature and the cpu is marked in the broadcast mask
  131. * then clear the broadcast bit.
  132. */
  133. if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
  134. int cpu = smp_processor_id();
  135. cpumask_clear_cpu(cpu, tick_broadcast_mask);
  136. tick_broadcast_clear_oneshot(cpu);
  137. } else {
  138. tick_device_setup_broadcast_func(dev);
  139. }
  140. }
  141. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  142. return ret;
  143. }
  144. #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
  145. int tick_receive_broadcast(void)
  146. {
  147. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  148. struct clock_event_device *evt = td->evtdev;
  149. if (!evt)
  150. return -ENODEV;
  151. if (!evt->event_handler)
  152. return -EINVAL;
  153. evt->event_handler(evt);
  154. return 0;
  155. }
  156. #endif
  157. /*
  158. * Broadcast the event to the cpus, which are set in the mask (mangled).
  159. */
  160. static void tick_do_broadcast(struct cpumask *mask)
  161. {
  162. int cpu = smp_processor_id();
  163. struct tick_device *td;
  164. /*
  165. * Check, if the current cpu is in the mask
  166. */
  167. if (cpumask_test_cpu(cpu, mask)) {
  168. cpumask_clear_cpu(cpu, mask);
  169. td = &per_cpu(tick_cpu_device, cpu);
  170. td->evtdev->event_handler(td->evtdev);
  171. }
  172. if (!cpumask_empty(mask)) {
  173. /*
  174. * It might be necessary to actually check whether the devices
  175. * have different broadcast functions. For now, just use the
  176. * one of the first device. This works as long as we have this
  177. * misfeature only on x86 (lapic)
  178. */
  179. td = &per_cpu(tick_cpu_device, cpumask_first(mask));
  180. td->evtdev->broadcast(mask);
  181. }
  182. }
  183. /*
  184. * Periodic broadcast:
  185. * - invoke the broadcast handlers
  186. */
  187. static void tick_do_periodic_broadcast(void)
  188. {
  189. raw_spin_lock(&tick_broadcast_lock);
  190. cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
  191. tick_do_broadcast(tmpmask);
  192. raw_spin_unlock(&tick_broadcast_lock);
  193. }
  194. /*
  195. * Event handler for periodic broadcast ticks
  196. */
  197. static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
  198. {
  199. ktime_t next;
  200. tick_do_periodic_broadcast();
  201. /*
  202. * The device is in periodic mode. No reprogramming necessary:
  203. */
  204. if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
  205. return;
  206. /*
  207. * Setup the next period for devices, which do not have
  208. * periodic mode. We read dev->next_event first and add to it
  209. * when the event already expired. clockevents_program_event()
  210. * sets dev->next_event only when the event is really
  211. * programmed to the device.
  212. */
  213. for (next = dev->next_event; ;) {
  214. next = ktime_add(next, tick_period);
  215. if (!clockevents_program_event(dev, next, false))
  216. return;
  217. tick_do_periodic_broadcast();
  218. }
  219. }
  220. /*
  221. * Powerstate information: The system enters/leaves a state, where
  222. * affected devices might stop
  223. */
  224. static void tick_do_broadcast_on_off(unsigned long *reason)
  225. {
  226. struct clock_event_device *bc, *dev;
  227. struct tick_device *td;
  228. unsigned long flags;
  229. int cpu, bc_stopped;
  230. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  231. cpu = smp_processor_id();
  232. td = &per_cpu(tick_cpu_device, cpu);
  233. dev = td->evtdev;
  234. bc = tick_broadcast_device.evtdev;
  235. /*
  236. * Is the device not affected by the powerstate ?
  237. */
  238. if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
  239. goto out;
  240. if (!tick_device_is_functional(dev))
  241. goto out;
  242. bc_stopped = cpumask_empty(tick_broadcast_mask);
  243. switch (*reason) {
  244. case CLOCK_EVT_NOTIFY_BROADCAST_ON:
  245. case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
  246. if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
  247. if (tick_broadcast_device.mode ==
  248. TICKDEV_MODE_PERIODIC)
  249. clockevents_shutdown(dev);
  250. }
  251. if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
  252. tick_broadcast_force = 1;
  253. break;
  254. case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
  255. if (!tick_broadcast_force &&
  256. cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
  257. if (tick_broadcast_device.mode ==
  258. TICKDEV_MODE_PERIODIC)
  259. tick_setup_periodic(dev, 0);
  260. }
  261. break;
  262. }
  263. if (cpumask_empty(tick_broadcast_mask)) {
  264. if (!bc_stopped)
  265. clockevents_shutdown(bc);
  266. } else if (bc_stopped) {
  267. if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
  268. tick_broadcast_start_periodic(bc);
  269. else
  270. tick_broadcast_setup_oneshot(bc);
  271. }
  272. out:
  273. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  274. }
  275. /*
  276. * Powerstate information: The system enters/leaves a state, where
  277. * affected devices might stop.
  278. */
  279. void tick_broadcast_on_off(unsigned long reason, int *oncpu)
  280. {
  281. if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
  282. printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
  283. "offline CPU #%d\n", *oncpu);
  284. else
  285. tick_do_broadcast_on_off(&reason);
  286. }
  287. /*
  288. * Set the periodic handler depending on broadcast on/off
  289. */
  290. void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
  291. {
  292. if (!broadcast)
  293. dev->event_handler = tick_handle_periodic;
  294. else
  295. dev->event_handler = tick_handle_periodic_broadcast;
  296. }
  297. /*
  298. * Remove a CPU from broadcasting
  299. */
  300. void tick_shutdown_broadcast(unsigned int *cpup)
  301. {
  302. struct clock_event_device *bc;
  303. unsigned long flags;
  304. unsigned int cpu = *cpup;
  305. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  306. bc = tick_broadcast_device.evtdev;
  307. cpumask_clear_cpu(cpu, tick_broadcast_mask);
  308. if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
  309. if (bc && cpumask_empty(tick_broadcast_mask))
  310. clockevents_shutdown(bc);
  311. }
  312. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  313. }
  314. void tick_suspend_broadcast(void)
  315. {
  316. struct clock_event_device *bc;
  317. unsigned long flags;
  318. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  319. bc = tick_broadcast_device.evtdev;
  320. if (bc)
  321. clockevents_shutdown(bc);
  322. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  323. }
  324. int tick_resume_broadcast(void)
  325. {
  326. struct clock_event_device *bc;
  327. unsigned long flags;
  328. int broadcast = 0;
  329. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  330. bc = tick_broadcast_device.evtdev;
  331. if (bc) {
  332. clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);
  333. switch (tick_broadcast_device.mode) {
  334. case TICKDEV_MODE_PERIODIC:
  335. if (!cpumask_empty(tick_broadcast_mask))
  336. tick_broadcast_start_periodic(bc);
  337. broadcast = cpumask_test_cpu(smp_processor_id(),
  338. tick_broadcast_mask);
  339. break;
  340. case TICKDEV_MODE_ONESHOT:
  341. if (!cpumask_empty(tick_broadcast_mask))
  342. broadcast = tick_resume_broadcast_oneshot(bc);
  343. break;
  344. }
  345. }
  346. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  347. return broadcast;
  348. }
  349. #ifdef CONFIG_TICK_ONESHOT
  350. static cpumask_var_t tick_broadcast_oneshot_mask;
  351. static cpumask_var_t tick_broadcast_pending_mask;
  352. static cpumask_var_t tick_broadcast_force_mask;
  353. /*
  354. * Exposed for debugging: see timer_list.c
  355. */
  356. struct cpumask *tick_get_broadcast_oneshot_mask(void)
  357. {
  358. return tick_broadcast_oneshot_mask;
  359. }
  360. /*
  361. * Called before going idle with interrupts disabled. Checks whether a
  362. * broadcast event from the other core is about to happen. We detected
  363. * that in tick_broadcast_oneshot_control(). The callsite can use this
  364. * to avoid a deep idle transition as we are about to get the
  365. * broadcast IPI right away.
  366. */
  367. int tick_check_broadcast_expired(void)
  368. {
  369. return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
  370. }
  371. /*
  372. * Set broadcast interrupt affinity
  373. */
  374. static void tick_broadcast_set_affinity(struct clock_event_device *bc,
  375. const struct cpumask *cpumask)
  376. {
  377. if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
  378. return;
  379. if (cpumask_equal(bc->cpumask, cpumask))
  380. return;
  381. bc->cpumask = cpumask;
  382. irq_set_affinity(bc->irq, bc->cpumask);
  383. }
  384. static int tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
  385. ktime_t expires, int force)
  386. {
  387. int ret;
  388. if (bc->mode != CLOCK_EVT_MODE_ONESHOT)
  389. clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
  390. ret = clockevents_program_event(bc, expires, force);
  391. if (!ret)
  392. tick_broadcast_set_affinity(bc, cpumask_of(cpu));
  393. return ret;
  394. }
  395. int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
  396. {
  397. clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
  398. return 0;
  399. }
  400. /*
  401. * Called from irq_enter() when idle was interrupted to reenable the
  402. * per cpu device.
  403. */
  404. void tick_check_oneshot_broadcast(int cpu)
  405. {
  406. if (cpumask_test_cpu(cpu, tick_broadcast_oneshot_mask)) {
  407. struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
  408. clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT);
  409. }
  410. }
  411. /*
  412. * Handle oneshot mode broadcasting
  413. */
  414. static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
  415. {
  416. struct tick_device *td;
  417. ktime_t now, next_event;
  418. int cpu, next_cpu = 0;
  419. raw_spin_lock(&tick_broadcast_lock);
  420. again:
  421. dev->next_event.tv64 = KTIME_MAX;
  422. next_event.tv64 = KTIME_MAX;
  423. cpumask_clear(tmpmask);
  424. now = ktime_get();
  425. /* Find all expired events */
  426. for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
  427. td = &per_cpu(tick_cpu_device, cpu);
  428. if (td->evtdev->next_event.tv64 <= now.tv64) {
  429. cpumask_set_cpu(cpu, tmpmask);
  430. /*
  431. * Mark the remote cpu in the pending mask, so
  432. * it can avoid reprogramming the cpu local
  433. * timer in tick_broadcast_oneshot_control().
  434. */
  435. cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
  436. } else if (td->evtdev->next_event.tv64 < next_event.tv64) {
  437. next_event.tv64 = td->evtdev->next_event.tv64;
  438. next_cpu = cpu;
  439. }
  440. }
  441. /*
  442. * Remove the current cpu from the pending mask. The event is
  443. * delivered immediately in tick_do_broadcast() !
  444. */
  445. cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
  446. /* Take care of enforced broadcast requests */
  447. cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
  448. cpumask_clear(tick_broadcast_force_mask);
  449. /*
  450. * Wakeup the cpus which have an expired event.
  451. */
  452. tick_do_broadcast(tmpmask);
  453. /*
  454. * Two reasons for reprogram:
  455. *
  456. * - The global event did not expire any CPU local
  457. * events. This happens in dyntick mode, as the maximum PIT
  458. * delta is quite small.
  459. *
  460. * - There are pending events on sleeping CPUs which were not
  461. * in the event mask
  462. */
  463. if (next_event.tv64 != KTIME_MAX) {
  464. /*
  465. * Rearm the broadcast device. If event expired,
  466. * repeat the above
  467. */
  468. if (tick_broadcast_set_event(dev, next_cpu, next_event, 0))
  469. goto again;
  470. }
  471. raw_spin_unlock(&tick_broadcast_lock);
  472. }
  473. /*
  474. * Powerstate information: The system enters/leaves a state, where
  475. * affected devices might stop
  476. */
  477. void tick_broadcast_oneshot_control(unsigned long reason)
  478. {
  479. struct clock_event_device *bc, *dev;
  480. struct tick_device *td;
  481. unsigned long flags;
  482. ktime_t now;
  483. int cpu;
  484. /*
  485. * Periodic mode does not care about the enter/exit of power
  486. * states
  487. */
  488. if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
  489. return;
  490. /*
  491. * We are called with preemtion disabled from the depth of the
  492. * idle code, so we can't be moved away.
  493. */
  494. cpu = smp_processor_id();
  495. td = &per_cpu(tick_cpu_device, cpu);
  496. dev = td->evtdev;
  497. if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
  498. return;
  499. bc = tick_broadcast_device.evtdev;
  500. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  501. if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
  502. if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
  503. WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
  504. clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
  505. /*
  506. * We only reprogram the broadcast timer if we
  507. * did not mark ourself in the force mask and
  508. * if the cpu local event is earlier than the
  509. * broadcast event. If the current CPU is in
  510. * the force mask, then we are going to be
  511. * woken by the IPI right away.
  512. */
  513. if (!cpumask_test_cpu(cpu, tick_broadcast_force_mask) &&
  514. dev->next_event.tv64 < bc->next_event.tv64)
  515. tick_broadcast_set_event(bc, cpu, dev->next_event, 1);
  516. }
  517. } else {
  518. if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
  519. clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
  520. if (dev->next_event.tv64 == KTIME_MAX)
  521. goto out;
  522. /*
  523. * The cpu which was handling the broadcast
  524. * timer marked this cpu in the broadcast
  525. * pending mask and fired the broadcast
  526. * IPI. So we are going to handle the expired
  527. * event anyway via the broadcast IPI
  528. * handler. No need to reprogram the timer
  529. * with an already expired event.
  530. */
  531. if (cpumask_test_and_clear_cpu(cpu,
  532. tick_broadcast_pending_mask))
  533. goto out;
  534. /*
  535. * If the pending bit is not set, then we are
  536. * either the CPU handling the broadcast
  537. * interrupt or we got woken by something else.
  538. *
  539. * We are not longer in the broadcast mask, so
  540. * if the cpu local expiry time is already
  541. * reached, we would reprogram the cpu local
  542. * timer with an already expired event.
  543. *
  544. * This can lead to a ping-pong when we return
  545. * to idle and therefor rearm the broadcast
  546. * timer before the cpu local timer was able
  547. * to fire. This happens because the forced
  548. * reprogramming makes sure that the event
  549. * will happen in the future and depending on
  550. * the min_delta setting this might be far
  551. * enough out that the ping-pong starts.
  552. *
  553. * If the cpu local next_event has expired
  554. * then we know that the broadcast timer
  555. * next_event has expired as well and
  556. * broadcast is about to be handled. So we
  557. * avoid reprogramming and enforce that the
  558. * broadcast handler, which did not run yet,
  559. * will invoke the cpu local handler.
  560. *
  561. * We cannot call the handler directly from
  562. * here, because we might be in a NOHZ phase
  563. * and we did not go through the irq_enter()
  564. * nohz fixups.
  565. */
  566. now = ktime_get();
  567. if (dev->next_event.tv64 <= now.tv64) {
  568. cpumask_set_cpu(cpu, tick_broadcast_force_mask);
  569. goto out;
  570. }
  571. /*
  572. * We got woken by something else. Reprogram
  573. * the cpu local timer device.
  574. */
  575. tick_program_event(dev->next_event, 1);
  576. }
  577. }
  578. out:
  579. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  580. }
  581. /*
  582. * Reset the one shot broadcast for a cpu
  583. *
  584. * Called with tick_broadcast_lock held
  585. */
  586. static void tick_broadcast_clear_oneshot(int cpu)
  587. {
  588. cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
  589. }
  590. static void tick_broadcast_init_next_event(struct cpumask *mask,
  591. ktime_t expires)
  592. {
  593. struct tick_device *td;
  594. int cpu;
  595. for_each_cpu(cpu, mask) {
  596. td = &per_cpu(tick_cpu_device, cpu);
  597. if (td->evtdev)
  598. td->evtdev->next_event = expires;
  599. }
  600. }
  601. /**
  602. * tick_broadcast_setup_oneshot - setup the broadcast device
  603. */
  604. void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
  605. {
  606. int cpu = smp_processor_id();
  607. /* Set it up only once ! */
  608. if (bc->event_handler != tick_handle_oneshot_broadcast) {
  609. int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;
  610. bc->event_handler = tick_handle_oneshot_broadcast;
  611. /* Take the do_timer update */
  612. if (!tick_nohz_full_cpu(cpu))
  613. tick_do_timer_cpu = cpu;
  614. /*
  615. * We must be careful here. There might be other CPUs
  616. * waiting for periodic broadcast. We need to set the
  617. * oneshot_mask bits for those and program the
  618. * broadcast device to fire.
  619. */
  620. cpumask_copy(tmpmask, tick_broadcast_mask);
  621. cpumask_clear_cpu(cpu, tmpmask);
  622. cpumask_or(tick_broadcast_oneshot_mask,
  623. tick_broadcast_oneshot_mask, tmpmask);
  624. if (was_periodic && !cpumask_empty(tmpmask)) {
  625. clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
  626. tick_broadcast_init_next_event(tmpmask,
  627. tick_next_period);
  628. tick_broadcast_set_event(bc, cpu, tick_next_period, 1);
  629. } else
  630. bc->next_event.tv64 = KTIME_MAX;
  631. } else {
  632. /*
  633. * The first cpu which switches to oneshot mode sets
  634. * the bit for all other cpus which are in the general
  635. * (periodic) broadcast mask. So the bit is set and
  636. * would prevent the first broadcast enter after this
  637. * to program the bc device.
  638. */
  639. tick_broadcast_clear_oneshot(cpu);
  640. }
  641. }
  642. /*
  643. * Select oneshot operating mode for the broadcast device
  644. */
  645. void tick_broadcast_switch_to_oneshot(void)
  646. {
  647. struct clock_event_device *bc;
  648. unsigned long flags;
  649. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  650. tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
  651. bc = tick_broadcast_device.evtdev;
  652. if (bc)
  653. tick_broadcast_setup_oneshot(bc);
  654. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  655. }
  656. /*
  657. * Remove a dead CPU from broadcasting
  658. */
  659. void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
  660. {
  661. unsigned long flags;
  662. unsigned int cpu = *cpup;
  663. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  664. /*
  665. * Clear the broadcast mask flag for the dead cpu, but do not
  666. * stop the broadcast device!
  667. */
  668. cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
  669. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  670. }
  671. /*
  672. * Check, whether the broadcast device is in one shot mode
  673. */
  674. int tick_broadcast_oneshot_active(void)
  675. {
  676. return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
  677. }
  678. /*
  679. * Check whether the broadcast device supports oneshot.
  680. */
  681. bool tick_broadcast_oneshot_available(void)
  682. {
  683. struct clock_event_device *bc = tick_broadcast_device.evtdev;
  684. return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
  685. }
  686. #endif
  687. void __init tick_broadcast_init(void)
  688. {
  689. zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
  690. zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
  691. #ifdef CONFIG_TICK_ONESHOT
  692. zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
  693. zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
  694. zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
  695. #endif
  696. }