sched_clock.c 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216
  1. /*
  2. * sched_clock.c: support for extending counters to full 64-bit ns counter
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License version 2 as
  6. * published by the Free Software Foundation.
  7. */
  8. #include <linux/clocksource.h>
  9. #include <linux/init.h>
  10. #include <linux/jiffies.h>
  11. #include <linux/ktime.h>
  12. #include <linux/kernel.h>
  13. #include <linux/moduleparam.h>
  14. #include <linux/sched.h>
  15. #include <linux/syscore_ops.h>
  16. #include <linux/hrtimer.h>
  17. #include <linux/sched_clock.h>
  18. #include <linux/seqlock.h>
  19. #include <linux/bitops.h>
  20. struct clock_data {
  21. ktime_t wrap_kt;
  22. u64 epoch_ns;
  23. u64 epoch_cyc;
  24. seqcount_t seq;
  25. unsigned long rate;
  26. u32 mult;
  27. u32 shift;
  28. bool suspended;
  29. };
  30. static struct hrtimer sched_clock_timer;
  31. static int irqtime = -1;
  32. core_param(irqtime, irqtime, int, 0400);
  33. static struct clock_data cd = {
  34. .mult = NSEC_PER_SEC / HZ,
  35. };
  36. static u64 __read_mostly sched_clock_mask;
  37. static u64 notrace jiffy_sched_clock_read(void)
  38. {
  39. /*
  40. * We don't need to use get_jiffies_64 on 32-bit arches here
  41. * because we register with BITS_PER_LONG
  42. */
  43. return (u64)(jiffies - INITIAL_JIFFIES);
  44. }
  45. static u32 __read_mostly (*read_sched_clock_32)(void);
  46. static u64 notrace read_sched_clock_32_wrapper(void)
  47. {
  48. return read_sched_clock_32();
  49. }
  50. static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read;
  51. static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift)
  52. {
  53. return (cyc * mult) >> shift;
  54. }
  55. unsigned long long notrace sched_clock(void)
  56. {
  57. u64 epoch_ns;
  58. u64 epoch_cyc;
  59. u64 cyc;
  60. unsigned long seq;
  61. if (cd.suspended)
  62. return cd.epoch_ns;
  63. do {
  64. seq = read_seqcount_begin(&cd.seq);
  65. epoch_cyc = cd.epoch_cyc;
  66. epoch_ns = cd.epoch_ns;
  67. } while (read_seqcount_retry(&cd.seq, seq));
  68. cyc = read_sched_clock();
  69. cyc = (cyc - epoch_cyc) & sched_clock_mask;
  70. return epoch_ns + cyc_to_ns(cyc, cd.mult, cd.shift);
  71. }
  72. /*
  73. * Atomically update the sched_clock epoch.
  74. */
  75. static void notrace update_sched_clock(void)
  76. {
  77. unsigned long flags;
  78. u64 cyc;
  79. u64 ns;
  80. cyc = read_sched_clock();
  81. ns = cd.epoch_ns +
  82. cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask,
  83. cd.mult, cd.shift);
  84. raw_local_irq_save(flags);
  85. write_seqcount_begin(&cd.seq);
  86. cd.epoch_ns = ns;
  87. cd.epoch_cyc = cyc;
  88. write_seqcount_end(&cd.seq);
  89. raw_local_irq_restore(flags);
  90. }
  91. static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)
  92. {
  93. update_sched_clock();
  94. hrtimer_forward_now(hrt, cd.wrap_kt);
  95. return HRTIMER_RESTART;
  96. }
  97. void __init sched_clock_register(u64 (*read)(void), int bits,
  98. unsigned long rate)
  99. {
  100. unsigned long r;
  101. u64 res, wrap;
  102. char r_unit;
  103. if (cd.rate > rate)
  104. return;
  105. WARN_ON(!irqs_disabled());
  106. read_sched_clock = read;
  107. sched_clock_mask = CLOCKSOURCE_MASK(bits);
  108. cd.rate = rate;
  109. /* calculate the mult/shift to convert counter ticks to ns. */
  110. clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 3600);
  111. r = rate;
  112. if (r >= 4000000) {
  113. r /= 1000000;
  114. r_unit = 'M';
  115. } else if (r >= 1000) {
  116. r /= 1000;
  117. r_unit = 'k';
  118. } else
  119. r_unit = ' ';
  120. /* calculate how many ns until we wrap */
  121. wrap = clocks_calc_max_nsecs(cd.mult, cd.shift, 0, sched_clock_mask);
  122. cd.wrap_kt = ns_to_ktime(wrap - (wrap >> 3));
  123. /* calculate the ns resolution of this counter */
  124. res = cyc_to_ns(1ULL, cd.mult, cd.shift);
  125. pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n",
  126. bits, r, r_unit, res, wrap);
  127. update_sched_clock();
  128. /*
  129. * Ensure that sched_clock() starts off at 0ns
  130. */
  131. cd.epoch_ns = 0;
  132. /* Enable IRQ time accounting if we have a fast enough sched_clock */
  133. if (irqtime > 0 || (irqtime == -1 && rate >= 1000000))
  134. enable_sched_clock_irqtime();
  135. pr_debug("Registered %pF as sched_clock source\n", read);
  136. }
  137. void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate)
  138. {
  139. read_sched_clock_32 = read;
  140. sched_clock_register(read_sched_clock_32_wrapper, bits, rate);
  141. }
  142. void __init sched_clock_postinit(void)
  143. {
  144. /*
  145. * If no sched_clock function has been provided at that point,
  146. * make it the final one one.
  147. */
  148. if (read_sched_clock == jiffy_sched_clock_read)
  149. sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ);
  150. update_sched_clock();
  151. /*
  152. * Start the timer to keep sched_clock() properly updated and
  153. * sets the initial epoch.
  154. */
  155. hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  156. sched_clock_timer.function = sched_clock_poll;
  157. hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL);
  158. }
  159. static int sched_clock_suspend(void)
  160. {
  161. sched_clock_poll(&sched_clock_timer);
  162. cd.suspended = true;
  163. return 0;
  164. }
  165. static void sched_clock_resume(void)
  166. {
  167. cd.epoch_cyc = read_sched_clock();
  168. cd.suspended = false;
  169. }
  170. static struct syscore_ops sched_clock_ops = {
  171. .suspend = sched_clock_suspend,
  172. .resume = sched_clock_resume,
  173. };
  174. static int __init sched_clock_syscore_init(void)
  175. {
  176. register_syscore_ops(&sched_clock_ops);
  177. return 0;
  178. }
  179. device_initcall(sched_clock_syscore_init);