sched_clock.c 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217
  1. /*
  2. * sched_clock.c: support for extending counters to full 64-bit ns counter
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License version 2 as
  6. * published by the Free Software Foundation.
  7. */
  8. #include <linux/clocksource.h>
  9. #include <linux/init.h>
  10. #include <linux/jiffies.h>
  11. #include <linux/kernel.h>
  12. #include <linux/moduleparam.h>
  13. #include <linux/sched.h>
  14. #include <linux/syscore_ops.h>
  15. #include <linux/timer.h>
  16. #include <asm/sched_clock.h>
  17. struct clock_data {
  18. u64 epoch_ns;
  19. u32 epoch_cyc;
  20. u32 epoch_cyc_copy;
  21. unsigned long rate;
  22. u32 mult;
  23. u32 shift;
  24. bool suspended;
  25. bool needs_suspend;
  26. };
  27. static void sched_clock_poll(unsigned long wrap_ticks);
  28. static DEFINE_TIMER(sched_clock_timer, sched_clock_poll, 0, 0);
  29. static int irqtime = -1;
  30. core_param(irqtime, irqtime, int, 0400);
  31. static struct clock_data cd = {
  32. .mult = NSEC_PER_SEC / HZ,
  33. };
  34. static u32 __read_mostly sched_clock_mask = 0xffffffff;
  35. static u32 notrace jiffy_sched_clock_read(void)
  36. {
  37. return (u32)(jiffies - INITIAL_JIFFIES);
  38. }
  39. static u32 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read;
  40. static inline u64 cyc_to_ns(u64 cyc, u32 mult, u32 shift)
  41. {
  42. return (cyc * mult) >> shift;
  43. }
  44. static unsigned long long cyc_to_sched_clock(u32 cyc, u32 mask)
  45. {
  46. u64 epoch_ns;
  47. u32 epoch_cyc;
  48. if (cd.suspended)
  49. return cd.epoch_ns;
  50. /*
  51. * Load the epoch_cyc and epoch_ns atomically. We do this by
  52. * ensuring that we always write epoch_cyc, epoch_ns and
  53. * epoch_cyc_copy in strict order, and read them in strict order.
  54. * If epoch_cyc and epoch_cyc_copy are not equal, then we're in
  55. * the middle of an update, and we should repeat the load.
  56. */
  57. do {
  58. epoch_cyc = cd.epoch_cyc;
  59. smp_rmb();
  60. epoch_ns = cd.epoch_ns;
  61. smp_rmb();
  62. } while (epoch_cyc != cd.epoch_cyc_copy);
  63. return epoch_ns + cyc_to_ns((cyc - epoch_cyc) & mask, cd.mult, cd.shift);
  64. }
  65. /*
  66. * Atomically update the sched_clock epoch.
  67. */
  68. static void notrace update_sched_clock(void)
  69. {
  70. unsigned long flags;
  71. u32 cyc;
  72. u64 ns;
  73. cyc = read_sched_clock();
  74. ns = cd.epoch_ns +
  75. cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask,
  76. cd.mult, cd.shift);
  77. /*
  78. * Write epoch_cyc and epoch_ns in a way that the update is
  79. * detectable in cyc_to_fixed_sched_clock().
  80. */
  81. raw_local_irq_save(flags);
  82. cd.epoch_cyc_copy = cyc;
  83. smp_wmb();
  84. cd.epoch_ns = ns;
  85. smp_wmb();
  86. cd.epoch_cyc = cyc;
  87. raw_local_irq_restore(flags);
  88. }
  89. static void sched_clock_poll(unsigned long wrap_ticks)
  90. {
  91. mod_timer(&sched_clock_timer, round_jiffies(jiffies + wrap_ticks));
  92. update_sched_clock();
  93. }
  94. void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate)
  95. {
  96. unsigned long r, w;
  97. u64 res, wrap;
  98. char r_unit;
  99. if (cd.rate > rate)
  100. return;
  101. BUG_ON(bits > 32);
  102. WARN_ON(!irqs_disabled());
  103. read_sched_clock = read;
  104. sched_clock_mask = (1 << bits) - 1;
  105. cd.rate = rate;
  106. /* calculate the mult/shift to convert counter ticks to ns. */
  107. clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 0);
  108. r = rate;
  109. if (r >= 4000000) {
  110. r /= 1000000;
  111. r_unit = 'M';
  112. } else if (r >= 1000) {
  113. r /= 1000;
  114. r_unit = 'k';
  115. } else
  116. r_unit = ' ';
  117. /* calculate how many ns until we wrap */
  118. wrap = cyc_to_ns((1ULL << bits) - 1, cd.mult, cd.shift);
  119. do_div(wrap, NSEC_PER_MSEC);
  120. w = wrap;
  121. /* calculate the ns resolution of this counter */
  122. res = cyc_to_ns(1ULL, cd.mult, cd.shift);
  123. pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lums\n",
  124. bits, r, r_unit, res, w);
  125. /*
  126. * Start the timer to keep sched_clock() properly updated and
  127. * sets the initial epoch.
  128. */
  129. sched_clock_timer.data = msecs_to_jiffies(w - (w / 10));
  130. update_sched_clock();
  131. /*
  132. * Ensure that sched_clock() starts off at 0ns
  133. */
  134. cd.epoch_ns = 0;
  135. /* Enable IRQ time accounting if we have a fast enough sched_clock */
  136. if (irqtime > 0 || (irqtime == -1 && rate >= 1000000))
  137. enable_sched_clock_irqtime();
  138. pr_debug("Registered %pF as sched_clock source\n", read);
  139. }
  140. static unsigned long long notrace sched_clock_32(void)
  141. {
  142. u32 cyc = read_sched_clock();
  143. return cyc_to_sched_clock(cyc, sched_clock_mask);
  144. }
  145. unsigned long long __read_mostly (*sched_clock_func)(void) = sched_clock_32;
  146. unsigned long long notrace sched_clock(void)
  147. {
  148. return sched_clock_func();
  149. }
  150. void __init sched_clock_postinit(void)
  151. {
  152. /*
  153. * If no sched_clock function has been provided at that point,
  154. * make it the final one one.
  155. */
  156. if (read_sched_clock == jiffy_sched_clock_read)
  157. setup_sched_clock(jiffy_sched_clock_read, 32, HZ);
  158. sched_clock_poll(sched_clock_timer.data);
  159. }
  160. static int sched_clock_suspend(void)
  161. {
  162. sched_clock_poll(sched_clock_timer.data);
  163. cd.suspended = true;
  164. return 0;
  165. }
  166. static void sched_clock_resume(void)
  167. {
  168. cd.epoch_cyc = read_sched_clock();
  169. cd.epoch_cyc_copy = cd.epoch_cyc;
  170. cd.suspended = false;
  171. }
  172. static struct syscore_ops sched_clock_ops = {
  173. .suspend = sched_clock_suspend,
  174. .resume = sched_clock_resume,
  175. };
  176. static int __init sched_clock_syscore_init(void)
  177. {
  178. register_syscore_ops(&sched_clock_ops);
  179. return 0;
  180. }
  181. device_initcall(sched_clock_syscore_init);