timekeeping.c 32 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234
  1. /*
  2. * linux/kernel/time/timekeeping.c
  3. *
  4. * Kernel timekeeping code and accessor functions
  5. *
  6. * This code was moved from linux/kernel/timer.c.
  7. * Please see that file for copyright and history logs.
  8. *
  9. */
  10. #include <linux/module.h>
  11. #include <linux/interrupt.h>
  12. #include <linux/percpu.h>
  13. #include <linux/init.h>
  14. #include <linux/mm.h>
  15. #include <linux/sched.h>
  16. #include <linux/syscore_ops.h>
  17. #include <linux/clocksource.h>
  18. #include <linux/jiffies.h>
  19. #include <linux/time.h>
  20. #include <linux/tick.h>
  21. #include <linux/stop_machine.h>
  22. /* Structure holding internal timekeeping values. */
  23. struct timekeeper {
  24. /* Current clocksource used for timekeeping. */
  25. struct clocksource *clock;
  26. /* The shift value of the current clocksource. */
  27. int shift;
  28. /* Number of clock cycles in one NTP interval. */
  29. cycle_t cycle_interval;
  30. /* Number of clock shifted nano seconds in one NTP interval. */
  31. u64 xtime_interval;
  32. /* shifted nano seconds left over when rounding cycle_interval */
  33. s64 xtime_remainder;
  34. /* Raw nano seconds accumulated per NTP interval. */
  35. u32 raw_interval;
  36. /* Clock shifted nano seconds remainder not stored in xtime.tv_nsec. */
  37. u64 xtime_nsec;
  38. /* Difference between accumulated time and NTP time in ntp
  39. * shifted nano seconds. */
  40. s64 ntp_error;
  41. /* Shift conversion between clock shifted nano seconds and
  42. * ntp shifted nano seconds. */
  43. int ntp_error_shift;
  44. /* NTP adjusted clock multiplier */
  45. u32 mult;
  46. };
  47. static struct timekeeper timekeeper;
  48. /**
  49. * timekeeper_setup_internals - Set up internals to use clocksource clock.
  50. *
  51. * @clock: Pointer to clocksource.
  52. *
  53. * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
  54. * pair and interval request.
  55. *
  56. * Unless you're the timekeeping code, you should not be using this!
  57. */
  58. static void timekeeper_setup_internals(struct clocksource *clock)
  59. {
  60. cycle_t interval;
  61. u64 tmp, ntpinterval;
  62. timekeeper.clock = clock;
  63. clock->cycle_last = clock->read(clock);
  64. /* Do the ns -> cycle conversion first, using original mult */
  65. tmp = NTP_INTERVAL_LENGTH;
  66. tmp <<= clock->shift;
  67. ntpinterval = tmp;
  68. tmp += clock->mult/2;
  69. do_div(tmp, clock->mult);
  70. if (tmp == 0)
  71. tmp = 1;
  72. interval = (cycle_t) tmp;
  73. timekeeper.cycle_interval = interval;
  74. /* Go back from cycles -> shifted ns */
  75. timekeeper.xtime_interval = (u64) interval * clock->mult;
  76. timekeeper.xtime_remainder = ntpinterval - timekeeper.xtime_interval;
  77. timekeeper.raw_interval =
  78. ((u64) interval * clock->mult) >> clock->shift;
  79. timekeeper.xtime_nsec = 0;
  80. timekeeper.shift = clock->shift;
  81. timekeeper.ntp_error = 0;
  82. timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
  83. /*
  84. * The timekeeper keeps its own mult values for the currently
  85. * active clocksource. These value will be adjusted via NTP
  86. * to counteract clock drifting.
  87. */
  88. timekeeper.mult = clock->mult;
  89. }
  90. /* Timekeeper helper functions. */
  91. static inline s64 timekeeping_get_ns(void)
  92. {
  93. cycle_t cycle_now, cycle_delta;
  94. struct clocksource *clock;
  95. /* read clocksource: */
  96. clock = timekeeper.clock;
  97. cycle_now = clock->read(clock);
  98. /* calculate the delta since the last update_wall_time: */
  99. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  100. /* return delta convert to nanoseconds using ntp adjusted mult. */
  101. return clocksource_cyc2ns(cycle_delta, timekeeper.mult,
  102. timekeeper.shift);
  103. }
  104. static inline s64 timekeeping_get_ns_raw(void)
  105. {
  106. cycle_t cycle_now, cycle_delta;
  107. struct clocksource *clock;
  108. /* read clocksource: */
  109. clock = timekeeper.clock;
  110. cycle_now = clock->read(clock);
  111. /* calculate the delta since the last update_wall_time: */
  112. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  113. /* return delta convert to nanoseconds using ntp adjusted mult. */
  114. return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
  115. }
  116. /*
  117. * This read-write spinlock protects us from races in SMP while
  118. * playing with xtime.
  119. */
  120. __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);
  121. /*
  122. * The current time
  123. * wall_to_monotonic is what we need to add to xtime (or xtime corrected
  124. * for sub jiffie times) to get to monotonic time. Monotonic is pegged
  125. * at zero at system boot time, so wall_to_monotonic will be negative,
  126. * however, we will ALWAYS keep the tv_nsec part positive so we can use
  127. * the usual normalization.
  128. *
  129. * wall_to_monotonic is moved after resume from suspend for the monotonic
  130. * time not to jump. We need to add total_sleep_time to wall_to_monotonic
  131. * to get the real boot based time offset.
  132. *
  133. * - wall_to_monotonic is no longer the boot time, getboottime must be
  134. * used instead.
  135. */
  136. static struct timespec xtime __attribute__ ((aligned (16)));
  137. static struct timespec wall_to_monotonic __attribute__ ((aligned (16)));
  138. static struct timespec total_sleep_time;
  139. /*
  140. * The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock.
  141. */
  142. static struct timespec raw_time;
  143. /* flag for if timekeeping is suspended */
  144. int __read_mostly timekeeping_suspended;
  145. /* must hold xtime_lock */
  146. void timekeeping_leap_insert(int leapsecond)
  147. {
  148. xtime.tv_sec += leapsecond;
  149. wall_to_monotonic.tv_sec -= leapsecond;
  150. update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
  151. timekeeper.mult);
  152. }
  153. /**
  154. * timekeeping_forward_now - update clock to the current time
  155. *
  156. * Forward the current clock to update its state since the last call to
  157. * update_wall_time(). This is useful before significant clock changes,
  158. * as it avoids having to deal with this time offset explicitly.
  159. */
  160. static void timekeeping_forward_now(void)
  161. {
  162. cycle_t cycle_now, cycle_delta;
  163. struct clocksource *clock;
  164. s64 nsec;
  165. clock = timekeeper.clock;
  166. cycle_now = clock->read(clock);
  167. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  168. clock->cycle_last = cycle_now;
  169. nsec = clocksource_cyc2ns(cycle_delta, timekeeper.mult,
  170. timekeeper.shift);
  171. /* If arch requires, add in gettimeoffset() */
  172. nsec += arch_gettimeoffset();
  173. timespec_add_ns(&xtime, nsec);
  174. nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
  175. timespec_add_ns(&raw_time, nsec);
  176. }
  177. /**
  178. * getnstimeofday - Returns the time of day in a timespec
  179. * @ts: pointer to the timespec to be set
  180. *
  181. * Returns the time of day in a timespec.
  182. */
  183. void getnstimeofday(struct timespec *ts)
  184. {
  185. unsigned long seq;
  186. s64 nsecs;
  187. WARN_ON(timekeeping_suspended);
  188. do {
  189. seq = read_seqbegin(&xtime_lock);
  190. *ts = xtime;
  191. nsecs = timekeeping_get_ns();
  192. /* If arch requires, add in gettimeoffset() */
  193. nsecs += arch_gettimeoffset();
  194. } while (read_seqretry(&xtime_lock, seq));
  195. timespec_add_ns(ts, nsecs);
  196. }
  197. EXPORT_SYMBOL(getnstimeofday);
  198. ktime_t ktime_get(void)
  199. {
  200. unsigned int seq;
  201. s64 secs, nsecs;
  202. WARN_ON(timekeeping_suspended);
  203. do {
  204. seq = read_seqbegin(&xtime_lock);
  205. secs = xtime.tv_sec + wall_to_monotonic.tv_sec;
  206. nsecs = xtime.tv_nsec + wall_to_monotonic.tv_nsec;
  207. nsecs += timekeeping_get_ns();
  208. } while (read_seqretry(&xtime_lock, seq));
  209. /*
  210. * Use ktime_set/ktime_add_ns to create a proper ktime on
  211. * 32-bit architectures without CONFIG_KTIME_SCALAR.
  212. */
  213. return ktime_add_ns(ktime_set(secs, 0), nsecs);
  214. }
  215. EXPORT_SYMBOL_GPL(ktime_get);
  216. /**
  217. * ktime_get_ts - get the monotonic clock in timespec format
  218. * @ts: pointer to timespec variable
  219. *
  220. * The function calculates the monotonic clock from the realtime
  221. * clock and the wall_to_monotonic offset and stores the result
  222. * in normalized timespec format in the variable pointed to by @ts.
  223. */
  224. void ktime_get_ts(struct timespec *ts)
  225. {
  226. struct timespec tomono;
  227. unsigned int seq;
  228. s64 nsecs;
  229. WARN_ON(timekeeping_suspended);
  230. do {
  231. seq = read_seqbegin(&xtime_lock);
  232. *ts = xtime;
  233. tomono = wall_to_monotonic;
  234. nsecs = timekeeping_get_ns();
  235. } while (read_seqretry(&xtime_lock, seq));
  236. set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
  237. ts->tv_nsec + tomono.tv_nsec + nsecs);
  238. }
  239. EXPORT_SYMBOL_GPL(ktime_get_ts);
  240. #ifdef CONFIG_NTP_PPS
  241. /**
  242. * getnstime_raw_and_real - get day and raw monotonic time in timespec format
  243. * @ts_raw: pointer to the timespec to be set to raw monotonic time
  244. * @ts_real: pointer to the timespec to be set to the time of day
  245. *
  246. * This function reads both the time of day and raw monotonic time at the
  247. * same time atomically and stores the resulting timestamps in timespec
  248. * format.
  249. */
  250. void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
  251. {
  252. unsigned long seq;
  253. s64 nsecs_raw, nsecs_real;
  254. WARN_ON_ONCE(timekeeping_suspended);
  255. do {
  256. u32 arch_offset;
  257. seq = read_seqbegin(&xtime_lock);
  258. *ts_raw = raw_time;
  259. *ts_real = xtime;
  260. nsecs_raw = timekeeping_get_ns_raw();
  261. nsecs_real = timekeeping_get_ns();
  262. /* If arch requires, add in gettimeoffset() */
  263. arch_offset = arch_gettimeoffset();
  264. nsecs_raw += arch_offset;
  265. nsecs_real += arch_offset;
  266. } while (read_seqretry(&xtime_lock, seq));
  267. timespec_add_ns(ts_raw, nsecs_raw);
  268. timespec_add_ns(ts_real, nsecs_real);
  269. }
  270. EXPORT_SYMBOL(getnstime_raw_and_real);
  271. #endif /* CONFIG_NTP_PPS */
  272. /**
  273. * do_gettimeofday - Returns the time of day in a timeval
  274. * @tv: pointer to the timeval to be set
  275. *
  276. * NOTE: Users should be converted to using getnstimeofday()
  277. */
  278. void do_gettimeofday(struct timeval *tv)
  279. {
  280. struct timespec now;
  281. getnstimeofday(&now);
  282. tv->tv_sec = now.tv_sec;
  283. tv->tv_usec = now.tv_nsec/1000;
  284. }
  285. EXPORT_SYMBOL(do_gettimeofday);
  286. /**
  287. * do_settimeofday - Sets the time of day
  288. * @tv: pointer to the timespec variable containing the new time
  289. *
  290. * Sets the time of day to the new time and update NTP and notify hrtimers
  291. */
  292. int do_settimeofday(const struct timespec *tv)
  293. {
  294. struct timespec ts_delta;
  295. unsigned long flags;
  296. if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
  297. return -EINVAL;
  298. write_seqlock_irqsave(&xtime_lock, flags);
  299. timekeeping_forward_now();
  300. ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec;
  301. ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec;
  302. wall_to_monotonic = timespec_sub(wall_to_monotonic, ts_delta);
  303. xtime = *tv;
  304. timekeeper.ntp_error = 0;
  305. ntp_clear();
  306. update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
  307. timekeeper.mult);
  308. write_sequnlock_irqrestore(&xtime_lock, flags);
  309. /* signal hrtimers about time change */
  310. clock_was_set();
  311. return 0;
  312. }
  313. EXPORT_SYMBOL(do_settimeofday);
  314. /**
  315. * timekeeping_inject_offset - Adds or subtracts from the current time.
  316. * @tv: pointer to the timespec variable containing the offset
  317. *
  318. * Adds or subtracts an offset value from the current time.
  319. */
  320. int timekeeping_inject_offset(struct timespec *ts)
  321. {
  322. unsigned long flags;
  323. if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
  324. return -EINVAL;
  325. write_seqlock_irqsave(&xtime_lock, flags);
  326. timekeeping_forward_now();
  327. xtime = timespec_add(xtime, *ts);
  328. wall_to_monotonic = timespec_sub(wall_to_monotonic, *ts);
  329. timekeeper.ntp_error = 0;
  330. ntp_clear();
  331. update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
  332. timekeeper.mult);
  333. write_sequnlock_irqrestore(&xtime_lock, flags);
  334. /* signal hrtimers about time change */
  335. clock_was_set();
  336. return 0;
  337. }
  338. EXPORT_SYMBOL(timekeeping_inject_offset);
  339. /**
  340. * change_clocksource - Swaps clocksources if a new one is available
  341. *
  342. * Accumulates current time interval and initializes new clocksource
  343. */
  344. static int change_clocksource(void *data)
  345. {
  346. struct clocksource *new, *old;
  347. new = (struct clocksource *) data;
  348. timekeeping_forward_now();
  349. if (!new->enable || new->enable(new) == 0) {
  350. old = timekeeper.clock;
  351. timekeeper_setup_internals(new);
  352. if (old->disable)
  353. old->disable(old);
  354. }
  355. return 0;
  356. }
  357. /**
  358. * timekeeping_notify - Install a new clock source
  359. * @clock: pointer to the clock source
  360. *
  361. * This function is called from clocksource.c after a new, better clock
  362. * source has been registered. The caller holds the clocksource_mutex.
  363. */
  364. void timekeeping_notify(struct clocksource *clock)
  365. {
  366. if (timekeeper.clock == clock)
  367. return;
  368. stop_machine(change_clocksource, clock, NULL);
  369. tick_clock_notify();
  370. }
  371. /**
  372. * ktime_get_real - get the real (wall-) time in ktime_t format
  373. *
  374. * returns the time in ktime_t format
  375. */
  376. ktime_t ktime_get_real(void)
  377. {
  378. struct timespec now;
  379. getnstimeofday(&now);
  380. return timespec_to_ktime(now);
  381. }
  382. EXPORT_SYMBOL_GPL(ktime_get_real);
  383. /**
  384. * getrawmonotonic - Returns the raw monotonic time in a timespec
  385. * @ts: pointer to the timespec to be set
  386. *
  387. * Returns the raw monotonic time (completely un-modified by ntp)
  388. */
  389. void getrawmonotonic(struct timespec *ts)
  390. {
  391. unsigned long seq;
  392. s64 nsecs;
  393. do {
  394. seq = read_seqbegin(&xtime_lock);
  395. nsecs = timekeeping_get_ns_raw();
  396. *ts = raw_time;
  397. } while (read_seqretry(&xtime_lock, seq));
  398. timespec_add_ns(ts, nsecs);
  399. }
  400. EXPORT_SYMBOL(getrawmonotonic);
  401. /**
  402. * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
  403. */
  404. int timekeeping_valid_for_hres(void)
  405. {
  406. unsigned long seq;
  407. int ret;
  408. do {
  409. seq = read_seqbegin(&xtime_lock);
  410. ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
  411. } while (read_seqretry(&xtime_lock, seq));
  412. return ret;
  413. }
  414. /**
  415. * timekeeping_max_deferment - Returns max time the clocksource can be deferred
  416. *
  417. * Caller must observe xtime_lock via read_seqbegin/read_seqretry to
  418. * ensure that the clocksource does not change!
  419. */
  420. u64 timekeeping_max_deferment(void)
  421. {
  422. return timekeeper.clock->max_idle_ns;
  423. }
  424. /**
  425. * read_persistent_clock - Return time from the persistent clock.
  426. *
  427. * Weak dummy function for arches that do not yet support it.
  428. * Reads the time from the battery backed persistent clock.
  429. * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
  430. *
  431. * XXX - Do be sure to remove it once all arches implement it.
  432. */
  433. void __attribute__((weak)) read_persistent_clock(struct timespec *ts)
  434. {
  435. ts->tv_sec = 0;
  436. ts->tv_nsec = 0;
  437. }
  438. /**
  439. * read_boot_clock - Return time of the system start.
  440. *
  441. * Weak dummy function for arches that do not yet support it.
  442. * Function to read the exact time the system has been started.
  443. * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
  444. *
  445. * XXX - Do be sure to remove it once all arches implement it.
  446. */
  447. void __attribute__((weak)) read_boot_clock(struct timespec *ts)
  448. {
  449. ts->tv_sec = 0;
  450. ts->tv_nsec = 0;
  451. }
  452. /*
  453. * timekeeping_init - Initializes the clocksource and common timekeeping values
  454. */
  455. void __init timekeeping_init(void)
  456. {
  457. struct clocksource *clock;
  458. unsigned long flags;
  459. struct timespec now, boot;
  460. read_persistent_clock(&now);
  461. read_boot_clock(&boot);
  462. write_seqlock_irqsave(&xtime_lock, flags);
  463. ntp_init();
  464. clock = clocksource_default_clock();
  465. if (clock->enable)
  466. clock->enable(clock);
  467. timekeeper_setup_internals(clock);
  468. xtime.tv_sec = now.tv_sec;
  469. xtime.tv_nsec = now.tv_nsec;
  470. raw_time.tv_sec = 0;
  471. raw_time.tv_nsec = 0;
  472. if (boot.tv_sec == 0 && boot.tv_nsec == 0) {
  473. boot.tv_sec = xtime.tv_sec;
  474. boot.tv_nsec = xtime.tv_nsec;
  475. }
  476. set_normalized_timespec(&wall_to_monotonic,
  477. -boot.tv_sec, -boot.tv_nsec);
  478. total_sleep_time.tv_sec = 0;
  479. total_sleep_time.tv_nsec = 0;
  480. write_sequnlock_irqrestore(&xtime_lock, flags);
  481. }
  482. /* time in seconds when suspend began */
  483. static struct timespec timekeeping_suspend_time;
  484. /**
  485. * __timekeeping_inject_sleeptime - Internal function to add sleep interval
  486. * @delta: pointer to a timespec delta value
  487. *
  488. * Takes a timespec offset measuring a suspend interval and properly
  489. * adds the sleep offset to the timekeeping variables.
  490. */
  491. static void __timekeeping_inject_sleeptime(struct timespec *delta)
  492. {
  493. if (!timespec_valid(delta)) {
  494. printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid "
  495. "sleep delta value!\n");
  496. return;
  497. }
  498. xtime = timespec_add(xtime, *delta);
  499. wall_to_monotonic = timespec_sub(wall_to_monotonic, *delta);
  500. total_sleep_time = timespec_add(total_sleep_time, *delta);
  501. }
  502. /**
  503. * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values
  504. * @delta: pointer to a timespec delta value
  505. *
  506. * This hook is for architectures that cannot support read_persistent_clock
  507. * because their RTC/persistent clock is only accessible when irqs are enabled.
  508. *
  509. * This function should only be called by rtc_resume(), and allows
  510. * a suspend offset to be injected into the timekeeping values.
  511. */
  512. void timekeeping_inject_sleeptime(struct timespec *delta)
  513. {
  514. unsigned long flags;
  515. struct timespec ts;
  516. /* Make sure we don't set the clock twice */
  517. read_persistent_clock(&ts);
  518. if (!(ts.tv_sec == 0 && ts.tv_nsec == 0))
  519. return;
  520. write_seqlock_irqsave(&xtime_lock, flags);
  521. timekeeping_forward_now();
  522. __timekeeping_inject_sleeptime(delta);
  523. timekeeper.ntp_error = 0;
  524. ntp_clear();
  525. update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
  526. timekeeper.mult);
  527. write_sequnlock_irqrestore(&xtime_lock, flags);
  528. /* signal hrtimers about time change */
  529. clock_was_set();
  530. }
  531. /**
  532. * timekeeping_resume - Resumes the generic timekeeping subsystem.
  533. *
  534. * This is for the generic clocksource timekeeping.
  535. * xtime/wall_to_monotonic/jiffies/etc are
  536. * still managed by arch specific suspend/resume code.
  537. */
  538. static void timekeeping_resume(void)
  539. {
  540. unsigned long flags;
  541. struct timespec ts;
  542. read_persistent_clock(&ts);
  543. clocksource_resume();
  544. write_seqlock_irqsave(&xtime_lock, flags);
  545. if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) {
  546. ts = timespec_sub(ts, timekeeping_suspend_time);
  547. __timekeeping_inject_sleeptime(&ts);
  548. }
  549. /* re-base the last cycle value */
  550. timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
  551. timekeeper.ntp_error = 0;
  552. timekeeping_suspended = 0;
  553. write_sequnlock_irqrestore(&xtime_lock, flags);
  554. touch_softlockup_watchdog();
  555. clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
  556. /* Resume hrtimers */
  557. hrtimers_resume();
  558. }
  559. static int timekeeping_suspend(void)
  560. {
  561. unsigned long flags;
  562. struct timespec delta, delta_delta;
  563. static struct timespec old_delta;
  564. read_persistent_clock(&timekeeping_suspend_time);
  565. write_seqlock_irqsave(&xtime_lock, flags);
  566. timekeeping_forward_now();
  567. timekeeping_suspended = 1;
  568. /*
  569. * To avoid drift caused by repeated suspend/resumes,
  570. * which each can add ~1 second drift error,
  571. * try to compensate so the difference in system time
  572. * and persistent_clock time stays close to constant.
  573. */
  574. delta = timespec_sub(xtime, timekeeping_suspend_time);
  575. delta_delta = timespec_sub(delta, old_delta);
  576. if (abs(delta_delta.tv_sec) >= 2) {
  577. /*
  578. * if delta_delta is too large, assume time correction
  579. * has occured and set old_delta to the current delta.
  580. */
  581. old_delta = delta;
  582. } else {
  583. /* Otherwise try to adjust old_system to compensate */
  584. timekeeping_suspend_time =
  585. timespec_add(timekeeping_suspend_time, delta_delta);
  586. }
  587. write_sequnlock_irqrestore(&xtime_lock, flags);
  588. clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
  589. clocksource_suspend();
  590. return 0;
  591. }
  592. /* sysfs resume/suspend bits for timekeeping */
  593. static struct syscore_ops timekeeping_syscore_ops = {
  594. .resume = timekeeping_resume,
  595. .suspend = timekeeping_suspend,
  596. };
  597. static int __init timekeeping_init_ops(void)
  598. {
  599. register_syscore_ops(&timekeeping_syscore_ops);
  600. return 0;
  601. }
  602. device_initcall(timekeeping_init_ops);
  603. /*
  604. * If the error is already larger, we look ahead even further
  605. * to compensate for late or lost adjustments.
  606. */
  607. static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
  608. s64 *offset)
  609. {
  610. s64 tick_error, i;
  611. u32 look_ahead, adj;
  612. s32 error2, mult;
  613. /*
  614. * Use the current error value to determine how much to look ahead.
  615. * The larger the error the slower we adjust for it to avoid problems
  616. * with losing too many ticks, otherwise we would overadjust and
  617. * produce an even larger error. The smaller the adjustment the
  618. * faster we try to adjust for it, as lost ticks can do less harm
  619. * here. This is tuned so that an error of about 1 msec is adjusted
  620. * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
  621. */
  622. error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
  623. error2 = abs(error2);
  624. for (look_ahead = 0; error2 > 0; look_ahead++)
  625. error2 >>= 2;
  626. /*
  627. * Now calculate the error in (1 << look_ahead) ticks, but first
  628. * remove the single look ahead already included in the error.
  629. */
  630. tick_error = tick_length >> (timekeeper.ntp_error_shift + 1);
  631. tick_error -= timekeeper.xtime_interval >> 1;
  632. error = ((error - tick_error) >> look_ahead) + tick_error;
  633. /* Finally calculate the adjustment shift value. */
  634. i = *interval;
  635. mult = 1;
  636. if (error < 0) {
  637. error = -error;
  638. *interval = -*interval;
  639. *offset = -*offset;
  640. mult = -1;
  641. }
  642. for (adj = 0; error > i; adj++)
  643. error >>= 1;
  644. *interval <<= adj;
  645. *offset <<= adj;
  646. return mult << adj;
  647. }
  648. /*
  649. * Adjust the multiplier to reduce the error value,
  650. * this is optimized for the most common adjustments of -1,0,1,
  651. * for other values we can do a bit more work.
  652. */
  653. static void timekeeping_adjust(s64 offset)
  654. {
  655. s64 error, interval = timekeeper.cycle_interval;
  656. int adj;
  657. /*
  658. * The point of this is to check if the error is greater then half
  659. * an interval.
  660. *
  661. * First we shift it down from NTP_SHIFT to clocksource->shifted nsecs.
  662. *
  663. * Note we subtract one in the shift, so that error is really error*2.
  664. * This "saves" dividing(shifting) intererval twice, but keeps the
  665. * (error > interval) comparision as still measuring if error is
  666. * larger then half an interval.
  667. *
  668. * Note: It does not "save" on aggrivation when reading the code.
  669. */
  670. error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1);
  671. if (error > interval) {
  672. /*
  673. * We now divide error by 4(via shift), which checks if
  674. * the error is greater then twice the interval.
  675. * If it is greater, we need a bigadjust, if its smaller,
  676. * we can adjust by 1.
  677. */
  678. error >>= 2;
  679. /*
  680. * XXX - In update_wall_time, we round up to the next
  681. * nanosecond, and store the amount rounded up into
  682. * the error. This causes the likely below to be unlikely.
  683. *
  684. * The properfix is to avoid rounding up by using
  685. * the high precision timekeeper.xtime_nsec instead of
  686. * xtime.tv_nsec everywhere. Fixing this will take some
  687. * time.
  688. */
  689. if (likely(error <= interval))
  690. adj = 1;
  691. else
  692. adj = timekeeping_bigadjust(error, &interval, &offset);
  693. } else if (error < -interval) {
  694. /* See comment above, this is just switched for the negative */
  695. error >>= 2;
  696. if (likely(error >= -interval)) {
  697. adj = -1;
  698. interval = -interval;
  699. offset = -offset;
  700. } else
  701. adj = timekeeping_bigadjust(error, &interval, &offset);
  702. } else /* No adjustment needed */
  703. return;
  704. /*
  705. * So the following can be confusing.
  706. *
  707. * To keep things simple, lets assume adj == 1 for now.
  708. *
  709. * When adj != 1, remember that the interval and offset values
  710. * have been appropriately scaled so the math is the same.
  711. *
  712. * The basic idea here is that we're increasing the multiplier
  713. * by one, this causes the xtime_interval to be incremented by
  714. * one cycle_interval. This is because:
  715. * xtime_interval = cycle_interval * mult
  716. * So if mult is being incremented by one:
  717. * xtime_interval = cycle_interval * (mult + 1)
  718. * Its the same as:
  719. * xtime_interval = (cycle_interval * mult) + cycle_interval
  720. * Which can be shortened to:
  721. * xtime_interval += cycle_interval
  722. *
  723. * So offset stores the non-accumulated cycles. Thus the current
  724. * time (in shifted nanoseconds) is:
  725. * now = (offset * adj) + xtime_nsec
  726. * Now, even though we're adjusting the clock frequency, we have
  727. * to keep time consistent. In other words, we can't jump back
  728. * in time, and we also want to avoid jumping forward in time.
  729. *
  730. * So given the same offset value, we need the time to be the same
  731. * both before and after the freq adjustment.
  732. * now = (offset * adj_1) + xtime_nsec_1
  733. * now = (offset * adj_2) + xtime_nsec_2
  734. * So:
  735. * (offset * adj_1) + xtime_nsec_1 =
  736. * (offset * adj_2) + xtime_nsec_2
  737. * And we know:
  738. * adj_2 = adj_1 + 1
  739. * So:
  740. * (offset * adj_1) + xtime_nsec_1 =
  741. * (offset * (adj_1+1)) + xtime_nsec_2
  742. * (offset * adj_1) + xtime_nsec_1 =
  743. * (offset * adj_1) + offset + xtime_nsec_2
  744. * Canceling the sides:
  745. * xtime_nsec_1 = offset + xtime_nsec_2
  746. * Which gives us:
  747. * xtime_nsec_2 = xtime_nsec_1 - offset
  748. * Which simplfies to:
  749. * xtime_nsec -= offset
  750. *
  751. * XXX - TODO: Doc ntp_error calculation.
  752. */
  753. timekeeper.mult += adj;
  754. timekeeper.xtime_interval += interval;
  755. timekeeper.xtime_nsec -= offset;
  756. timekeeper.ntp_error -= (interval - offset) <<
  757. timekeeper.ntp_error_shift;
  758. }
  759. /**
  760. * logarithmic_accumulation - shifted accumulation of cycles
  761. *
  762. * This functions accumulates a shifted interval of cycles into
  763. * into a shifted interval nanoseconds. Allows for O(log) accumulation
  764. * loop.
  765. *
  766. * Returns the unconsumed cycles.
  767. */
  768. static cycle_t logarithmic_accumulation(cycle_t offset, int shift)
  769. {
  770. u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift;
  771. u64 raw_nsecs;
  772. /* If the offset is smaller then a shifted interval, do nothing */
  773. if (offset < timekeeper.cycle_interval<<shift)
  774. return offset;
  775. /* Accumulate one shifted interval */
  776. offset -= timekeeper.cycle_interval << shift;
  777. timekeeper.clock->cycle_last += timekeeper.cycle_interval << shift;
  778. timekeeper.xtime_nsec += timekeeper.xtime_interval << shift;
  779. while (timekeeper.xtime_nsec >= nsecps) {
  780. timekeeper.xtime_nsec -= nsecps;
  781. xtime.tv_sec++;
  782. second_overflow();
  783. }
  784. /* Accumulate raw time */
  785. raw_nsecs = timekeeper.raw_interval << shift;
  786. raw_nsecs += raw_time.tv_nsec;
  787. if (raw_nsecs >= NSEC_PER_SEC) {
  788. u64 raw_secs = raw_nsecs;
  789. raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
  790. raw_time.tv_sec += raw_secs;
  791. }
  792. raw_time.tv_nsec = raw_nsecs;
  793. /* Accumulate error between NTP and clock interval */
  794. timekeeper.ntp_error += tick_length << shift;
  795. timekeeper.ntp_error -=
  796. (timekeeper.xtime_interval + timekeeper.xtime_remainder) <<
  797. (timekeeper.ntp_error_shift + shift);
  798. return offset;
  799. }
  800. /**
  801. * update_wall_time - Uses the current clocksource to increment the wall time
  802. *
  803. * Called from the timer interrupt, must hold a write on xtime_lock.
  804. */
  805. static void update_wall_time(void)
  806. {
  807. struct clocksource *clock;
  808. cycle_t offset;
  809. int shift = 0, maxshift;
  810. /* Make sure we're fully resumed: */
  811. if (unlikely(timekeeping_suspended))
  812. return;
  813. clock = timekeeper.clock;
  814. #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
  815. offset = timekeeper.cycle_interval;
  816. #else
  817. offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
  818. #endif
  819. timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift;
  820. /*
  821. * With NO_HZ we may have to accumulate many cycle_intervals
  822. * (think "ticks") worth of time at once. To do this efficiently,
  823. * we calculate the largest doubling multiple of cycle_intervals
  824. * that is smaller then the offset. We then accumulate that
  825. * chunk in one go, and then try to consume the next smaller
  826. * doubled multiple.
  827. */
  828. shift = ilog2(offset) - ilog2(timekeeper.cycle_interval);
  829. shift = max(0, shift);
  830. /* Bound shift to one less then what overflows tick_length */
  831. maxshift = (8*sizeof(tick_length) - (ilog2(tick_length)+1)) - 1;
  832. shift = min(shift, maxshift);
  833. while (offset >= timekeeper.cycle_interval) {
  834. offset = logarithmic_accumulation(offset, shift);
  835. if(offset < timekeeper.cycle_interval<<shift)
  836. shift--;
  837. }
  838. /* correct the clock when NTP error is too big */
  839. timekeeping_adjust(offset);
  840. /*
  841. * Since in the loop above, we accumulate any amount of time
  842. * in xtime_nsec over a second into xtime.tv_sec, its possible for
  843. * xtime_nsec to be fairly small after the loop. Further, if we're
  844. * slightly speeding the clocksource up in timekeeping_adjust(),
  845. * its possible the required corrective factor to xtime_nsec could
  846. * cause it to underflow.
  847. *
  848. * Now, we cannot simply roll the accumulated second back, since
  849. * the NTP subsystem has been notified via second_overflow. So
  850. * instead we push xtime_nsec forward by the amount we underflowed,
  851. * and add that amount into the error.
  852. *
  853. * We'll correct this error next time through this function, when
  854. * xtime_nsec is not as small.
  855. */
  856. if (unlikely((s64)timekeeper.xtime_nsec < 0)) {
  857. s64 neg = -(s64)timekeeper.xtime_nsec;
  858. timekeeper.xtime_nsec = 0;
  859. timekeeper.ntp_error += neg << timekeeper.ntp_error_shift;
  860. }
  861. /*
  862. * Store full nanoseconds into xtime after rounding it up and
  863. * add the remainder to the error difference.
  864. */
  865. xtime.tv_nsec = ((s64) timekeeper.xtime_nsec >> timekeeper.shift) + 1;
  866. timekeeper.xtime_nsec -= (s64) xtime.tv_nsec << timekeeper.shift;
  867. timekeeper.ntp_error += timekeeper.xtime_nsec <<
  868. timekeeper.ntp_error_shift;
  869. /*
  870. * Finally, make sure that after the rounding
  871. * xtime.tv_nsec isn't larger then NSEC_PER_SEC
  872. */
  873. if (unlikely(xtime.tv_nsec >= NSEC_PER_SEC)) {
  874. xtime.tv_nsec -= NSEC_PER_SEC;
  875. xtime.tv_sec++;
  876. second_overflow();
  877. }
  878. /* check to see if there is a new clocksource to use */
  879. update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
  880. timekeeper.mult);
  881. }
  882. /**
  883. * getboottime - Return the real time of system boot.
  884. * @ts: pointer to the timespec to be set
  885. *
  886. * Returns the wall-time of boot in a timespec.
  887. *
  888. * This is based on the wall_to_monotonic offset and the total suspend
  889. * time. Calls to settimeofday will affect the value returned (which
  890. * basically means that however wrong your real time clock is at boot time,
  891. * you get the right time here).
  892. */
  893. void getboottime(struct timespec *ts)
  894. {
  895. struct timespec boottime = {
  896. .tv_sec = wall_to_monotonic.tv_sec + total_sleep_time.tv_sec,
  897. .tv_nsec = wall_to_monotonic.tv_nsec + total_sleep_time.tv_nsec
  898. };
  899. set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec);
  900. }
  901. EXPORT_SYMBOL_GPL(getboottime);
  902. /**
  903. * get_monotonic_boottime - Returns monotonic time since boot
  904. * @ts: pointer to the timespec to be set
  905. *
  906. * Returns the monotonic time since boot in a timespec.
  907. *
  908. * This is similar to CLOCK_MONTONIC/ktime_get_ts, but also
  909. * includes the time spent in suspend.
  910. */
  911. void get_monotonic_boottime(struct timespec *ts)
  912. {
  913. struct timespec tomono, sleep;
  914. unsigned int seq;
  915. s64 nsecs;
  916. WARN_ON(timekeeping_suspended);
  917. do {
  918. seq = read_seqbegin(&xtime_lock);
  919. *ts = xtime;
  920. tomono = wall_to_monotonic;
  921. sleep = total_sleep_time;
  922. nsecs = timekeeping_get_ns();
  923. } while (read_seqretry(&xtime_lock, seq));
  924. set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec,
  925. ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec + nsecs);
  926. }
  927. EXPORT_SYMBOL_GPL(get_monotonic_boottime);
  928. /**
  929. * ktime_get_boottime - Returns monotonic time since boot in a ktime
  930. *
  931. * Returns the monotonic time since boot in a ktime
  932. *
  933. * This is similar to CLOCK_MONTONIC/ktime_get, but also
  934. * includes the time spent in suspend.
  935. */
  936. ktime_t ktime_get_boottime(void)
  937. {
  938. struct timespec ts;
  939. get_monotonic_boottime(&ts);
  940. return timespec_to_ktime(ts);
  941. }
  942. EXPORT_SYMBOL_GPL(ktime_get_boottime);
  943. /**
  944. * monotonic_to_bootbased - Convert the monotonic time to boot based.
  945. * @ts: pointer to the timespec to be converted
  946. */
  947. void monotonic_to_bootbased(struct timespec *ts)
  948. {
  949. *ts = timespec_add(*ts, total_sleep_time);
  950. }
  951. EXPORT_SYMBOL_GPL(monotonic_to_bootbased);
  952. unsigned long get_seconds(void)
  953. {
  954. return xtime.tv_sec;
  955. }
  956. EXPORT_SYMBOL(get_seconds);
  957. struct timespec __current_kernel_time(void)
  958. {
  959. return xtime;
  960. }
  961. struct timespec current_kernel_time(void)
  962. {
  963. struct timespec now;
  964. unsigned long seq;
  965. do {
  966. seq = read_seqbegin(&xtime_lock);
  967. now = xtime;
  968. } while (read_seqretry(&xtime_lock, seq));
  969. return now;
  970. }
  971. EXPORT_SYMBOL(current_kernel_time);
  972. struct timespec get_monotonic_coarse(void)
  973. {
  974. struct timespec now, mono;
  975. unsigned long seq;
  976. do {
  977. seq = read_seqbegin(&xtime_lock);
  978. now = xtime;
  979. mono = wall_to_monotonic;
  980. } while (read_seqretry(&xtime_lock, seq));
  981. set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
  982. now.tv_nsec + mono.tv_nsec);
  983. return now;
  984. }
  985. /*
  986. * The 64-bit jiffies value is not atomic - you MUST NOT read it
  987. * without sampling the sequence number in xtime_lock.
  988. * jiffies is defined in the linker script...
  989. */
  990. void do_timer(unsigned long ticks)
  991. {
  992. jiffies_64 += ticks;
  993. update_wall_time();
  994. calc_global_load(ticks);
  995. }
  996. /**
  997. * get_xtime_and_monotonic_and_sleep_offset() - get xtime, wall_to_monotonic,
  998. * and sleep offsets.
  999. * @xtim: pointer to timespec to be set with xtime
  1000. * @wtom: pointer to timespec to be set with wall_to_monotonic
  1001. * @sleep: pointer to timespec to be set with time in suspend
  1002. */
  1003. void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
  1004. struct timespec *wtom, struct timespec *sleep)
  1005. {
  1006. unsigned long seq;
  1007. do {
  1008. seq = read_seqbegin(&xtime_lock);
  1009. *xtim = xtime;
  1010. *wtom = wall_to_monotonic;
  1011. *sleep = total_sleep_time;
  1012. } while (read_seqretry(&xtime_lock, seq));
  1013. }
  1014. /**
  1015. * ktime_get_monotonic_offset() - get wall_to_monotonic in ktime_t format
  1016. */
  1017. ktime_t ktime_get_monotonic_offset(void)
  1018. {
  1019. unsigned long seq;
  1020. struct timespec wtom;
  1021. do {
  1022. seq = read_seqbegin(&xtime_lock);
  1023. wtom = wall_to_monotonic;
  1024. } while (read_seqretry(&xtime_lock, seq));
  1025. return timespec_to_ktime(wtom);
  1026. }
  1027. /**
  1028. * xtime_update() - advances the timekeeping infrastructure
  1029. * @ticks: number of ticks, that have elapsed since the last call.
  1030. *
  1031. * Must be called with interrupts disabled.
  1032. */
  1033. void xtime_update(unsigned long ticks)
  1034. {
  1035. write_seqlock(&xtime_lock);
  1036. do_timer(ticks);
  1037. write_sequnlock(&xtime_lock);
  1038. }