tick-broadcast.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780
  1. /*
  2. * linux/kernel/time/tick-broadcast.c
  3. *
  4. * This file contains functions which emulate a local clock-event
  5. * device via a broadcast event source.
  6. *
  7. * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
  8. * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
  9. * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
  10. *
  11. * This code is licenced under the GPL version 2. For details see
  12. * kernel-base/COPYING.
  13. */
  14. #include <linux/cpu.h>
  15. #include <linux/err.h>
  16. #include <linux/hrtimer.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/percpu.h>
  19. #include <linux/profile.h>
  20. #include <linux/sched.h>
  21. #include <linux/smp.h>
  22. #include "tick-internal.h"
  23. /*
  24. * Broadcast support for broken x86 hardware, where the local apic
  25. * timer stops in C3 state.
  26. */
  27. static struct tick_device tick_broadcast_device;
  28. static cpumask_var_t tick_broadcast_mask;
  29. static cpumask_var_t tmpmask;
  30. static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
  31. static int tick_broadcast_force;
  32. #ifdef CONFIG_TICK_ONESHOT
  33. static void tick_broadcast_clear_oneshot(int cpu);
  34. #else
  35. static inline void tick_broadcast_clear_oneshot(int cpu) { }
  36. #endif
  37. /*
  38. * Debugging: see timer_list.c
  39. */
  40. struct tick_device *tick_get_broadcast_device(void)
  41. {
  42. return &tick_broadcast_device;
  43. }
  44. struct cpumask *tick_get_broadcast_mask(void)
  45. {
  46. return tick_broadcast_mask;
  47. }
  48. /*
  49. * Start the device in periodic mode
  50. */
  51. static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  52. {
  53. if (bc)
  54. tick_setup_periodic(bc, 1);
  55. }
  56. /*
  57. * Check, if the device can be utilized as broadcast device:
  58. */
  59. int tick_check_broadcast_device(struct clock_event_device *dev)
  60. {
  61. if ((tick_broadcast_device.evtdev &&
  62. tick_broadcast_device.evtdev->rating >= dev->rating) ||
  63. (dev->features & CLOCK_EVT_FEAT_C3STOP))
  64. return 0;
  65. clockevents_exchange_device(tick_broadcast_device.evtdev, dev);
  66. tick_broadcast_device.evtdev = dev;
  67. if (!cpumask_empty(tick_broadcast_mask))
  68. tick_broadcast_start_periodic(dev);
  69. return 1;
  70. }
  71. /*
  72. * Check, if the device is the broadcast device
  73. */
  74. int tick_is_broadcast_device(struct clock_event_device *dev)
  75. {
  76. return (dev && tick_broadcast_device.evtdev == dev);
  77. }
  78. static void err_broadcast(const struct cpumask *mask)
  79. {
  80. pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
  81. }
  82. static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
  83. {
  84. if (!dev->broadcast)
  85. dev->broadcast = tick_broadcast;
  86. if (!dev->broadcast) {
  87. pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
  88. dev->name);
  89. dev->broadcast = err_broadcast;
  90. }
  91. }
  92. /*
  93. * Check, if the device is disfunctional and a place holder, which
  94. * needs to be handled by the broadcast device.
  95. */
  96. int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
  97. {
  98. unsigned long flags;
  99. int ret = 0;
  100. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  101. /*
  102. * Devices might be registered with both periodic and oneshot
  103. * mode disabled. This signals, that the device needs to be
  104. * operated from the broadcast device and is a placeholder for
  105. * the cpu local device.
  106. */
  107. if (!tick_device_is_functional(dev)) {
  108. dev->event_handler = tick_handle_periodic;
  109. tick_device_setup_broadcast_func(dev);
  110. cpumask_set_cpu(cpu, tick_broadcast_mask);
  111. tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
  112. ret = 1;
  113. } else {
  114. /*
  115. * When the new device is not affected by the stop
  116. * feature and the cpu is marked in the broadcast mask
  117. * then clear the broadcast bit.
  118. */
  119. if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
  120. int cpu = smp_processor_id();
  121. cpumask_clear_cpu(cpu, tick_broadcast_mask);
  122. tick_broadcast_clear_oneshot(cpu);
  123. } else {
  124. tick_device_setup_broadcast_func(dev);
  125. }
  126. }
  127. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  128. return ret;
  129. }
  130. #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
  131. int tick_receive_broadcast(void)
  132. {
  133. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  134. struct clock_event_device *evt = td->evtdev;
  135. if (!evt)
  136. return -ENODEV;
  137. if (!evt->event_handler)
  138. return -EINVAL;
  139. evt->event_handler(evt);
  140. return 0;
  141. }
  142. #endif
  143. /*
  144. * Broadcast the event to the cpus, which are set in the mask (mangled).
  145. */
  146. static void tick_do_broadcast(struct cpumask *mask)
  147. {
  148. int cpu = smp_processor_id();
  149. struct tick_device *td;
  150. /*
  151. * Check, if the current cpu is in the mask
  152. */
  153. if (cpumask_test_cpu(cpu, mask)) {
  154. cpumask_clear_cpu(cpu, mask);
  155. td = &per_cpu(tick_cpu_device, cpu);
  156. td->evtdev->event_handler(td->evtdev);
  157. }
  158. if (!cpumask_empty(mask)) {
  159. /*
  160. * It might be necessary to actually check whether the devices
  161. * have different broadcast functions. For now, just use the
  162. * one of the first device. This works as long as we have this
  163. * misfeature only on x86 (lapic)
  164. */
  165. td = &per_cpu(tick_cpu_device, cpumask_first(mask));
  166. td->evtdev->broadcast(mask);
  167. }
  168. }
  169. /*
  170. * Periodic broadcast:
  171. * - invoke the broadcast handlers
  172. */
  173. static void tick_do_periodic_broadcast(void)
  174. {
  175. raw_spin_lock(&tick_broadcast_lock);
  176. cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
  177. tick_do_broadcast(tmpmask);
  178. raw_spin_unlock(&tick_broadcast_lock);
  179. }
  180. /*
  181. * Event handler for periodic broadcast ticks
  182. */
  183. static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
  184. {
  185. ktime_t next;
  186. tick_do_periodic_broadcast();
  187. /*
  188. * The device is in periodic mode. No reprogramming necessary:
  189. */
  190. if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
  191. return;
  192. /*
  193. * Setup the next period for devices, which do not have
  194. * periodic mode. We read dev->next_event first and add to it
  195. * when the event already expired. clockevents_program_event()
  196. * sets dev->next_event only when the event is really
  197. * programmed to the device.
  198. */
  199. for (next = dev->next_event; ;) {
  200. next = ktime_add(next, tick_period);
  201. if (!clockevents_program_event(dev, next, false))
  202. return;
  203. tick_do_periodic_broadcast();
  204. }
  205. }
  206. /*
  207. * Powerstate information: The system enters/leaves a state, where
  208. * affected devices might stop
  209. */
  210. static void tick_do_broadcast_on_off(unsigned long *reason)
  211. {
  212. struct clock_event_device *bc, *dev;
  213. struct tick_device *td;
  214. unsigned long flags;
  215. int cpu, bc_stopped;
  216. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  217. cpu = smp_processor_id();
  218. td = &per_cpu(tick_cpu_device, cpu);
  219. dev = td->evtdev;
  220. bc = tick_broadcast_device.evtdev;
  221. /*
  222. * Is the device not affected by the powerstate ?
  223. */
  224. if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
  225. goto out;
  226. if (!tick_device_is_functional(dev))
  227. goto out;
  228. bc_stopped = cpumask_empty(tick_broadcast_mask);
  229. switch (*reason) {
  230. case CLOCK_EVT_NOTIFY_BROADCAST_ON:
  231. case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
  232. if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
  233. if (tick_broadcast_device.mode ==
  234. TICKDEV_MODE_PERIODIC)
  235. clockevents_shutdown(dev);
  236. }
  237. if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
  238. tick_broadcast_force = 1;
  239. break;
  240. case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
  241. if (!tick_broadcast_force &&
  242. cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
  243. if (tick_broadcast_device.mode ==
  244. TICKDEV_MODE_PERIODIC)
  245. tick_setup_periodic(dev, 0);
  246. }
  247. break;
  248. }
  249. if (cpumask_empty(tick_broadcast_mask)) {
  250. if (!bc_stopped)
  251. clockevents_shutdown(bc);
  252. } else if (bc_stopped) {
  253. if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
  254. tick_broadcast_start_periodic(bc);
  255. else
  256. tick_broadcast_setup_oneshot(bc);
  257. }
  258. out:
  259. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  260. }
  261. /*
  262. * Powerstate information: The system enters/leaves a state, where
  263. * affected devices might stop.
  264. */
  265. void tick_broadcast_on_off(unsigned long reason, int *oncpu)
  266. {
  267. if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
  268. printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
  269. "offline CPU #%d\n", *oncpu);
  270. else
  271. tick_do_broadcast_on_off(&reason);
  272. }
  273. /*
  274. * Set the periodic handler depending on broadcast on/off
  275. */
  276. void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
  277. {
  278. if (!broadcast)
  279. dev->event_handler = tick_handle_periodic;
  280. else
  281. dev->event_handler = tick_handle_periodic_broadcast;
  282. }
  283. /*
  284. * Remove a CPU from broadcasting
  285. */
  286. void tick_shutdown_broadcast(unsigned int *cpup)
  287. {
  288. struct clock_event_device *bc;
  289. unsigned long flags;
  290. unsigned int cpu = *cpup;
  291. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  292. bc = tick_broadcast_device.evtdev;
  293. cpumask_clear_cpu(cpu, tick_broadcast_mask);
  294. if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
  295. if (bc && cpumask_empty(tick_broadcast_mask))
  296. clockevents_shutdown(bc);
  297. }
  298. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  299. }
  300. void tick_suspend_broadcast(void)
  301. {
  302. struct clock_event_device *bc;
  303. unsigned long flags;
  304. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  305. bc = tick_broadcast_device.evtdev;
  306. if (bc)
  307. clockevents_shutdown(bc);
  308. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  309. }
  310. int tick_resume_broadcast(void)
  311. {
  312. struct clock_event_device *bc;
  313. unsigned long flags;
  314. int broadcast = 0;
  315. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  316. bc = tick_broadcast_device.evtdev;
  317. if (bc) {
  318. clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);
  319. switch (tick_broadcast_device.mode) {
  320. case TICKDEV_MODE_PERIODIC:
  321. if (!cpumask_empty(tick_broadcast_mask))
  322. tick_broadcast_start_periodic(bc);
  323. broadcast = cpumask_test_cpu(smp_processor_id(),
  324. tick_broadcast_mask);
  325. break;
  326. case TICKDEV_MODE_ONESHOT:
  327. if (!cpumask_empty(tick_broadcast_mask))
  328. broadcast = tick_resume_broadcast_oneshot(bc);
  329. break;
  330. }
  331. }
  332. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  333. return broadcast;
  334. }
  335. #ifdef CONFIG_TICK_ONESHOT
  336. static cpumask_var_t tick_broadcast_oneshot_mask;
  337. static cpumask_var_t tick_broadcast_pending_mask;
  338. static cpumask_var_t tick_broadcast_force_mask;
  339. /*
  340. * Exposed for debugging: see timer_list.c
  341. */
  342. struct cpumask *tick_get_broadcast_oneshot_mask(void)
  343. {
  344. return tick_broadcast_oneshot_mask;
  345. }
  346. /*
  347. * Called before going idle with interrupts disabled. Checks whether a
  348. * broadcast event from the other core is about to happen. We detected
  349. * that in tick_broadcast_oneshot_control(). The callsite can use this
  350. * to avoid a deep idle transition as we are about to get the
  351. * broadcast IPI right away.
  352. */
  353. int tick_check_broadcast_expired(void)
  354. {
  355. return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
  356. }
  357. /*
  358. * Set broadcast interrupt affinity
  359. */
  360. static void tick_broadcast_set_affinity(struct clock_event_device *bc,
  361. const struct cpumask *cpumask)
  362. {
  363. if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
  364. return;
  365. if (cpumask_equal(bc->cpumask, cpumask))
  366. return;
  367. bc->cpumask = cpumask;
  368. irq_set_affinity(bc->irq, bc->cpumask);
  369. }
  370. static int tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
  371. ktime_t expires, int force)
  372. {
  373. int ret;
  374. if (bc->mode != CLOCK_EVT_MODE_ONESHOT)
  375. clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
  376. ret = clockevents_program_event(bc, expires, force);
  377. if (!ret)
  378. tick_broadcast_set_affinity(bc, cpumask_of(cpu));
  379. return ret;
  380. }
  381. int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
  382. {
  383. clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
  384. return 0;
  385. }
  386. /*
  387. * Called from irq_enter() when idle was interrupted to reenable the
  388. * per cpu device.
  389. */
  390. void tick_check_oneshot_broadcast(int cpu)
  391. {
  392. if (cpumask_test_cpu(cpu, tick_broadcast_oneshot_mask)) {
  393. struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
  394. clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT);
  395. }
  396. }
  397. /*
  398. * Handle oneshot mode broadcasting
  399. */
  400. static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
  401. {
  402. struct tick_device *td;
  403. ktime_t now, next_event;
  404. int cpu, next_cpu = 0;
  405. raw_spin_lock(&tick_broadcast_lock);
  406. again:
  407. dev->next_event.tv64 = KTIME_MAX;
  408. next_event.tv64 = KTIME_MAX;
  409. cpumask_clear(tmpmask);
  410. now = ktime_get();
  411. /* Find all expired events */
  412. for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
  413. td = &per_cpu(tick_cpu_device, cpu);
  414. if (td->evtdev->next_event.tv64 <= now.tv64) {
  415. cpumask_set_cpu(cpu, tmpmask);
  416. /*
  417. * Mark the remote cpu in the pending mask, so
  418. * it can avoid reprogramming the cpu local
  419. * timer in tick_broadcast_oneshot_control().
  420. */
  421. cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
  422. } else if (td->evtdev->next_event.tv64 < next_event.tv64) {
  423. next_event.tv64 = td->evtdev->next_event.tv64;
  424. next_cpu = cpu;
  425. }
  426. }
  427. /* Take care of enforced broadcast requests */
  428. cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
  429. cpumask_clear(tick_broadcast_force_mask);
  430. /*
  431. * Wakeup the cpus which have an expired event.
  432. */
  433. tick_do_broadcast(tmpmask);
  434. /*
  435. * Two reasons for reprogram:
  436. *
  437. * - The global event did not expire any CPU local
  438. * events. This happens in dyntick mode, as the maximum PIT
  439. * delta is quite small.
  440. *
  441. * - There are pending events on sleeping CPUs which were not
  442. * in the event mask
  443. */
  444. if (next_event.tv64 != KTIME_MAX) {
  445. /*
  446. * Rearm the broadcast device. If event expired,
  447. * repeat the above
  448. */
  449. if (tick_broadcast_set_event(dev, next_cpu, next_event, 0))
  450. goto again;
  451. }
  452. raw_spin_unlock(&tick_broadcast_lock);
  453. }
  454. /*
  455. * Powerstate information: The system enters/leaves a state, where
  456. * affected devices might stop
  457. */
  458. void tick_broadcast_oneshot_control(unsigned long reason)
  459. {
  460. struct clock_event_device *bc, *dev;
  461. struct tick_device *td;
  462. unsigned long flags;
  463. ktime_t now;
  464. int cpu;
  465. /*
  466. * Periodic mode does not care about the enter/exit of power
  467. * states
  468. */
  469. if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
  470. return;
  471. /*
  472. * We are called with preemtion disabled from the depth of the
  473. * idle code, so we can't be moved away.
  474. */
  475. cpu = smp_processor_id();
  476. td = &per_cpu(tick_cpu_device, cpu);
  477. dev = td->evtdev;
  478. if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
  479. return;
  480. bc = tick_broadcast_device.evtdev;
  481. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  482. if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
  483. WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
  484. if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
  485. clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
  486. /*
  487. * We only reprogram the broadcast timer if we
  488. * did not mark ourself in the force mask and
  489. * if the cpu local event is earlier than the
  490. * broadcast event. If the current CPU is in
  491. * the force mask, then we are going to be
  492. * woken by the IPI right away.
  493. */
  494. if (!cpumask_test_cpu(cpu, tick_broadcast_force_mask) &&
  495. dev->next_event.tv64 < bc->next_event.tv64)
  496. tick_broadcast_set_event(bc, cpu, dev->next_event, 1);
  497. }
  498. } else {
  499. if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
  500. clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
  501. if (dev->next_event.tv64 == KTIME_MAX)
  502. goto out;
  503. /*
  504. * The cpu which was handling the broadcast
  505. * timer marked this cpu in the broadcast
  506. * pending mask and fired the broadcast
  507. * IPI. So we are going to handle the expired
  508. * event anyway via the broadcast IPI
  509. * handler. No need to reprogram the timer
  510. * with an already expired event.
  511. */
  512. if (cpumask_test_and_clear_cpu(cpu,
  513. tick_broadcast_pending_mask))
  514. goto out;
  515. /*
  516. * If the pending bit is not set, then we are
  517. * either the CPU handling the broadcast
  518. * interrupt or we got woken by something else.
  519. *
  520. * We are not longer in the broadcast mask, so
  521. * if the cpu local expiry time is already
  522. * reached, we would reprogram the cpu local
  523. * timer with an already expired event.
  524. *
  525. * This can lead to a ping-pong when we return
  526. * to idle and therefor rearm the broadcast
  527. * timer before the cpu local timer was able
  528. * to fire. This happens because the forced
  529. * reprogramming makes sure that the event
  530. * will happen in the future and depending on
  531. * the min_delta setting this might be far
  532. * enough out that the ping-pong starts.
  533. *
  534. * If the cpu local next_event has expired
  535. * then we know that the broadcast timer
  536. * next_event has expired as well and
  537. * broadcast is about to be handled. So we
  538. * avoid reprogramming and enforce that the
  539. * broadcast handler, which did not run yet,
  540. * will invoke the cpu local handler.
  541. *
  542. * We cannot call the handler directly from
  543. * here, because we might be in a NOHZ phase
  544. * and we did not go through the irq_enter()
  545. * nohz fixups.
  546. */
  547. now = ktime_get();
  548. if (dev->next_event.tv64 <= now.tv64) {
  549. cpumask_set_cpu(cpu, tick_broadcast_force_mask);
  550. goto out;
  551. }
  552. /*
  553. * We got woken by something else. Reprogram
  554. * the cpu local timer device.
  555. */
  556. tick_program_event(dev->next_event, 1);
  557. }
  558. }
  559. out:
  560. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  561. }
  562. /*
  563. * Reset the one shot broadcast for a cpu
  564. *
  565. * Called with tick_broadcast_lock held
  566. */
  567. static void tick_broadcast_clear_oneshot(int cpu)
  568. {
  569. cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
  570. }
  571. static void tick_broadcast_init_next_event(struct cpumask *mask,
  572. ktime_t expires)
  573. {
  574. struct tick_device *td;
  575. int cpu;
  576. for_each_cpu(cpu, mask) {
  577. td = &per_cpu(tick_cpu_device, cpu);
  578. if (td->evtdev)
  579. td->evtdev->next_event = expires;
  580. }
  581. }
  582. /**
  583. * tick_broadcast_setup_oneshot - setup the broadcast device
  584. */
  585. void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
  586. {
  587. int cpu = smp_processor_id();
  588. /* Set it up only once ! */
  589. if (bc->event_handler != tick_handle_oneshot_broadcast) {
  590. int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;
  591. bc->event_handler = tick_handle_oneshot_broadcast;
  592. /* Take the do_timer update */
  593. tick_do_timer_cpu = cpu;
  594. /*
  595. * We must be careful here. There might be other CPUs
  596. * waiting for periodic broadcast. We need to set the
  597. * oneshot_mask bits for those and program the
  598. * broadcast device to fire.
  599. */
  600. cpumask_copy(tmpmask, tick_broadcast_mask);
  601. cpumask_clear_cpu(cpu, tmpmask);
  602. cpumask_or(tick_broadcast_oneshot_mask,
  603. tick_broadcast_oneshot_mask, tmpmask);
  604. if (was_periodic && !cpumask_empty(tmpmask)) {
  605. clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
  606. tick_broadcast_init_next_event(tmpmask,
  607. tick_next_period);
  608. tick_broadcast_set_event(bc, cpu, tick_next_period, 1);
  609. } else
  610. bc->next_event.tv64 = KTIME_MAX;
  611. } else {
  612. /*
  613. * The first cpu which switches to oneshot mode sets
  614. * the bit for all other cpus which are in the general
  615. * (periodic) broadcast mask. So the bit is set and
  616. * would prevent the first broadcast enter after this
  617. * to program the bc device.
  618. */
  619. tick_broadcast_clear_oneshot(cpu);
  620. }
  621. }
  622. /*
  623. * Select oneshot operating mode for the broadcast device
  624. */
  625. void tick_broadcast_switch_to_oneshot(void)
  626. {
  627. struct clock_event_device *bc;
  628. unsigned long flags;
  629. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  630. tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
  631. bc = tick_broadcast_device.evtdev;
  632. if (bc)
  633. tick_broadcast_setup_oneshot(bc);
  634. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  635. }
  636. /*
  637. * Remove a dead CPU from broadcasting
  638. */
  639. void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
  640. {
  641. unsigned long flags;
  642. unsigned int cpu = *cpup;
  643. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  644. /*
  645. * Clear the broadcast mask flag for the dead cpu, but do not
  646. * stop the broadcast device!
  647. */
  648. cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
  649. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  650. }
  651. /*
  652. * Check, whether the broadcast device is in one shot mode
  653. */
  654. int tick_broadcast_oneshot_active(void)
  655. {
  656. return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
  657. }
  658. /*
  659. * Check whether the broadcast device supports oneshot.
  660. */
  661. bool tick_broadcast_oneshot_available(void)
  662. {
  663. struct clock_event_device *bc = tick_broadcast_device.evtdev;
  664. return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
  665. }
  666. #endif
  667. void __init tick_broadcast_init(void)
  668. {
  669. alloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
  670. alloc_cpumask_var(&tmpmask, GFP_NOWAIT);
  671. #ifdef CONFIG_TICK_ONESHOT
  672. alloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
  673. alloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
  674. alloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
  675. #endif
  676. }