sched_clock.c 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216
  1. /*
  2. * sched_clock.c: support for extending counters to full 64-bit ns counter
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License version 2 as
  6. * published by the Free Software Foundation.
  7. */
  8. #include <linux/clocksource.h>
  9. #include <linux/init.h>
  10. #include <linux/jiffies.h>
  11. #include <linux/kernel.h>
  12. #include <linux/moduleparam.h>
  13. #include <linux/sched.h>
  14. #include <linux/syscore_ops.h>
  15. #include <linux/timer.h>
  16. #include <asm/sched_clock.h>
  17. struct clock_data {
  18. u64 epoch_ns;
  19. u32 epoch_cyc;
  20. u32 epoch_cyc_copy;
  21. u32 mult;
  22. u32 shift;
  23. bool suspended;
  24. bool needs_suspend;
  25. };
  26. static void sched_clock_poll(unsigned long wrap_ticks);
  27. static DEFINE_TIMER(sched_clock_timer, sched_clock_poll, 0, 0);
  28. static int irqtime = -1;
  29. core_param(irqtime, irqtime, int, 0400);
  30. static struct clock_data cd = {
  31. .mult = NSEC_PER_SEC / HZ,
  32. };
  33. static u32 __read_mostly sched_clock_mask = 0xffffffff;
  34. static u32 notrace jiffy_sched_clock_read(void)
  35. {
  36. return (u32)(jiffies - INITIAL_JIFFIES);
  37. }
  38. static u32 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read;
  39. static inline u64 cyc_to_ns(u64 cyc, u32 mult, u32 shift)
  40. {
  41. return (cyc * mult) >> shift;
  42. }
  43. static unsigned long long cyc_to_sched_clock(u32 cyc, u32 mask)
  44. {
  45. u64 epoch_ns;
  46. u32 epoch_cyc;
  47. if (cd.suspended)
  48. return cd.epoch_ns;
  49. /*
  50. * Load the epoch_cyc and epoch_ns atomically. We do this by
  51. * ensuring that we always write epoch_cyc, epoch_ns and
  52. * epoch_cyc_copy in strict order, and read them in strict order.
  53. * If epoch_cyc and epoch_cyc_copy are not equal, then we're in
  54. * the middle of an update, and we should repeat the load.
  55. */
  56. do {
  57. epoch_cyc = cd.epoch_cyc;
  58. smp_rmb();
  59. epoch_ns = cd.epoch_ns;
  60. smp_rmb();
  61. } while (epoch_cyc != cd.epoch_cyc_copy);
  62. return epoch_ns + cyc_to_ns((cyc - epoch_cyc) & mask, cd.mult, cd.shift);
  63. }
  64. /*
  65. * Atomically update the sched_clock epoch.
  66. */
  67. static void notrace update_sched_clock(void)
  68. {
  69. unsigned long flags;
  70. u32 cyc;
  71. u64 ns;
  72. cyc = read_sched_clock();
  73. ns = cd.epoch_ns +
  74. cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask,
  75. cd.mult, cd.shift);
  76. /*
  77. * Write epoch_cyc and epoch_ns in a way that the update is
  78. * detectable in cyc_to_fixed_sched_clock().
  79. */
  80. raw_local_irq_save(flags);
  81. cd.epoch_cyc = cyc;
  82. smp_wmb();
  83. cd.epoch_ns = ns;
  84. smp_wmb();
  85. cd.epoch_cyc_copy = cyc;
  86. raw_local_irq_restore(flags);
  87. }
  88. static void sched_clock_poll(unsigned long wrap_ticks)
  89. {
  90. mod_timer(&sched_clock_timer, round_jiffies(jiffies + wrap_ticks));
  91. update_sched_clock();
  92. }
  93. void __init setup_sched_clock_needs_suspend(u32 (*read)(void), int bits,
  94. unsigned long rate)
  95. {
  96. setup_sched_clock(read, bits, rate);
  97. cd.needs_suspend = true;
  98. }
  99. void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate)
  100. {
  101. unsigned long r, w;
  102. u64 res, wrap;
  103. char r_unit;
  104. BUG_ON(bits > 32);
  105. WARN_ON(!irqs_disabled());
  106. WARN_ON(read_sched_clock != jiffy_sched_clock_read);
  107. read_sched_clock = read;
  108. sched_clock_mask = (1 << bits) - 1;
  109. /* calculate the mult/shift to convert counter ticks to ns. */
  110. clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 0);
  111. r = rate;
  112. if (r >= 4000000) {
  113. r /= 1000000;
  114. r_unit = 'M';
  115. } else if (r >= 1000) {
  116. r /= 1000;
  117. r_unit = 'k';
  118. } else
  119. r_unit = ' ';
  120. /* calculate how many ns until we wrap */
  121. wrap = cyc_to_ns((1ULL << bits) - 1, cd.mult, cd.shift);
  122. do_div(wrap, NSEC_PER_MSEC);
  123. w = wrap;
  124. /* calculate the ns resolution of this counter */
  125. res = cyc_to_ns(1ULL, cd.mult, cd.shift);
  126. pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lums\n",
  127. bits, r, r_unit, res, w);
  128. /*
  129. * Start the timer to keep sched_clock() properly updated and
  130. * sets the initial epoch.
  131. */
  132. sched_clock_timer.data = msecs_to_jiffies(w - (w / 10));
  133. update_sched_clock();
  134. /*
  135. * Ensure that sched_clock() starts off at 0ns
  136. */
  137. cd.epoch_ns = 0;
  138. /* Enable IRQ time accounting if we have a fast enough sched_clock */
  139. if (irqtime > 0 || (irqtime == -1 && rate >= 1000000))
  140. enable_sched_clock_irqtime();
  141. pr_debug("Registered %pF as sched_clock source\n", read);
  142. }
  143. unsigned long long notrace sched_clock(void)
  144. {
  145. u32 cyc = read_sched_clock();
  146. return cyc_to_sched_clock(cyc, sched_clock_mask);
  147. }
  148. void __init sched_clock_postinit(void)
  149. {
  150. /*
  151. * If no sched_clock function has been provided at that point,
  152. * make it the final one one.
  153. */
  154. if (read_sched_clock == jiffy_sched_clock_read)
  155. setup_sched_clock(jiffy_sched_clock_read, 32, HZ);
  156. sched_clock_poll(sched_clock_timer.data);
  157. }
  158. static int sched_clock_suspend(void)
  159. {
  160. sched_clock_poll(sched_clock_timer.data);
  161. if (cd.needs_suspend)
  162. cd.suspended = true;
  163. return 0;
  164. }
  165. static void sched_clock_resume(void)
  166. {
  167. if (cd.needs_suspend) {
  168. cd.epoch_cyc = read_sched_clock();
  169. cd.epoch_cyc_copy = cd.epoch_cyc;
  170. cd.suspended = false;
  171. }
  172. }
  173. static struct syscore_ops sched_clock_ops = {
  174. .suspend = sched_clock_suspend,
  175. .resume = sched_clock_resume,
  176. };
  177. static int __init sched_clock_syscore_init(void)
  178. {
  179. register_syscore_ops(&sched_clock_ops);
  180. return 0;
  181. }
  182. device_initcall(sched_clock_syscore_init);