timekeeping.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847
  1. /*
  2. * linux/kernel/time/timekeeping.c
  3. *
  4. * Kernel timekeeping code and accessor functions
  5. *
  6. * This code was moved from linux/kernel/timer.c.
  7. * Please see that file for copyright and history logs.
  8. *
  9. */
  10. #include <linux/module.h>
  11. #include <linux/interrupt.h>
  12. #include <linux/percpu.h>
  13. #include <linux/init.h>
  14. #include <linux/mm.h>
  15. #include <linux/sysdev.h>
  16. #include <linux/clocksource.h>
  17. #include <linux/jiffies.h>
  18. #include <linux/time.h>
  19. #include <linux/tick.h>
  20. /* Structure holding internal timekeeping values. */
  21. struct timekeeper {
  22. /* Current clocksource used for timekeeping. */
  23. struct clocksource *clock;
  24. /* The shift value of the current clocksource. */
  25. int shift;
  26. /* Number of clock cycles in one NTP interval. */
  27. cycle_t cycle_interval;
  28. /* Number of clock shifted nano seconds in one NTP interval. */
  29. u64 xtime_interval;
  30. /* Raw nano seconds accumulated per NTP interval. */
  31. u32 raw_interval;
  32. /* Clock shifted nano seconds remainder not stored in xtime.tv_nsec. */
  33. u64 xtime_nsec;
  34. /* Difference between accumulated time and NTP time in ntp
  35. * shifted nano seconds. */
  36. s64 ntp_error;
  37. /* Shift conversion between clock shifted nano seconds and
  38. * ntp shifted nano seconds. */
  39. int ntp_error_shift;
  40. };
  41. struct timekeeper timekeeper;
  42. /**
  43. * timekeeper_setup_internals - Set up internals to use clocksource clock.
  44. *
  45. * @clock: Pointer to clocksource.
  46. *
  47. * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
  48. * pair and interval request.
  49. *
  50. * Unless you're the timekeeping code, you should not be using this!
  51. */
  52. static void timekeeper_setup_internals(struct clocksource *clock)
  53. {
  54. cycle_t interval;
  55. u64 tmp;
  56. timekeeper.clock = clock;
  57. clock->cycle_last = clock->read(clock);
  58. /* Do the ns -> cycle conversion first, using original mult */
  59. tmp = NTP_INTERVAL_LENGTH;
  60. tmp <<= clock->shift;
  61. tmp += clock->mult_orig/2;
  62. do_div(tmp, clock->mult_orig);
  63. if (tmp == 0)
  64. tmp = 1;
  65. interval = (cycle_t) tmp;
  66. timekeeper.cycle_interval = interval;
  67. /* Go back from cycles -> shifted ns */
  68. timekeeper.xtime_interval = (u64) interval * clock->mult;
  69. timekeeper.raw_interval =
  70. ((u64) interval * clock->mult_orig) >> clock->shift;
  71. timekeeper.xtime_nsec = 0;
  72. timekeeper.shift = clock->shift;
  73. timekeeper.ntp_error = 0;
  74. timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
  75. }
  76. /*
  77. * This read-write spinlock protects us from races in SMP while
  78. * playing with xtime.
  79. */
  80. __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);
  81. /*
  82. * The current time
  83. * wall_to_monotonic is what we need to add to xtime (or xtime corrected
  84. * for sub jiffie times) to get to monotonic time. Monotonic is pegged
  85. * at zero at system boot time, so wall_to_monotonic will be negative,
  86. * however, we will ALWAYS keep the tv_nsec part positive so we can use
  87. * the usual normalization.
  88. *
  89. * wall_to_monotonic is moved after resume from suspend for the monotonic
  90. * time not to jump. We need to add total_sleep_time to wall_to_monotonic
  91. * to get the real boot based time offset.
  92. *
  93. * - wall_to_monotonic is no longer the boot time, getboottime must be
  94. * used instead.
  95. */
  96. struct timespec xtime __attribute__ ((aligned (16)));
  97. struct timespec wall_to_monotonic __attribute__ ((aligned (16)));
  98. static unsigned long total_sleep_time; /* seconds */
  99. /*
  100. * The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock.
  101. */
  102. struct timespec raw_time;
  103. /* flag for if timekeeping is suspended */
  104. int __read_mostly timekeeping_suspended;
  105. static struct timespec xtime_cache __attribute__ ((aligned (16)));
  106. void update_xtime_cache(u64 nsec)
  107. {
  108. xtime_cache = xtime;
  109. timespec_add_ns(&xtime_cache, nsec);
  110. }
  111. /* must hold xtime_lock */
  112. void timekeeping_leap_insert(int leapsecond)
  113. {
  114. xtime.tv_sec += leapsecond;
  115. wall_to_monotonic.tv_sec -= leapsecond;
  116. update_vsyscall(&xtime, timekeeper.clock);
  117. }
  118. #ifdef CONFIG_GENERIC_TIME
  119. /**
  120. * timekeeping_forward_now - update clock to the current time
  121. *
  122. * Forward the current clock to update its state since the last call to
  123. * update_wall_time(). This is useful before significant clock changes,
  124. * as it avoids having to deal with this time offset explicitly.
  125. */
  126. static void timekeeping_forward_now(void)
  127. {
  128. cycle_t cycle_now, cycle_delta;
  129. struct clocksource *clock;
  130. s64 nsec;
  131. clock = timekeeper.clock;
  132. cycle_now = clock->read(clock);
  133. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  134. clock->cycle_last = cycle_now;
  135. nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
  136. /* If arch requires, add in gettimeoffset() */
  137. nsec += arch_gettimeoffset();
  138. timespec_add_ns(&xtime, nsec);
  139. nsec = clocksource_cyc2ns(cycle_delta, clock->mult_orig, clock->shift);
  140. timespec_add_ns(&raw_time, nsec);
  141. }
  142. /**
  143. * getnstimeofday - Returns the time of day in a timespec
  144. * @ts: pointer to the timespec to be set
  145. *
  146. * Returns the time of day in a timespec.
  147. */
  148. void getnstimeofday(struct timespec *ts)
  149. {
  150. cycle_t cycle_now, cycle_delta;
  151. struct clocksource *clock;
  152. unsigned long seq;
  153. s64 nsecs;
  154. WARN_ON(timekeeping_suspended);
  155. do {
  156. seq = read_seqbegin(&xtime_lock);
  157. *ts = xtime;
  158. /* read clocksource: */
  159. clock = timekeeper.clock;
  160. cycle_now = clock->read(clock);
  161. /* calculate the delta since the last update_wall_time: */
  162. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  163. /* convert to nanoseconds: */
  164. nsecs = clocksource_cyc2ns(cycle_delta, clock->mult,
  165. clock->shift);
  166. /* If arch requires, add in gettimeoffset() */
  167. nsecs += arch_gettimeoffset();
  168. } while (read_seqretry(&xtime_lock, seq));
  169. timespec_add_ns(ts, nsecs);
  170. }
  171. EXPORT_SYMBOL(getnstimeofday);
  172. ktime_t ktime_get(void)
  173. {
  174. cycle_t cycle_now, cycle_delta;
  175. struct clocksource *clock;
  176. unsigned int seq;
  177. s64 secs, nsecs;
  178. WARN_ON(timekeeping_suspended);
  179. do {
  180. seq = read_seqbegin(&xtime_lock);
  181. secs = xtime.tv_sec + wall_to_monotonic.tv_sec;
  182. nsecs = xtime.tv_nsec + wall_to_monotonic.tv_nsec;
  183. /* read clocksource: */
  184. clock = timekeeper.clock;
  185. cycle_now = clock->read(clock);
  186. /* calculate the delta since the last update_wall_time: */
  187. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  188. /* convert to nanoseconds: */
  189. nsecs += clocksource_cyc2ns(cycle_delta, clock->mult,
  190. clock->shift);
  191. } while (read_seqretry(&xtime_lock, seq));
  192. /*
  193. * Use ktime_set/ktime_add_ns to create a proper ktime on
  194. * 32-bit architectures without CONFIG_KTIME_SCALAR.
  195. */
  196. return ktime_add_ns(ktime_set(secs, 0), nsecs);
  197. }
  198. EXPORT_SYMBOL_GPL(ktime_get);
  199. /**
  200. * ktime_get_ts - get the monotonic clock in timespec format
  201. * @ts: pointer to timespec variable
  202. *
  203. * The function calculates the monotonic clock from the realtime
  204. * clock and the wall_to_monotonic offset and stores the result
  205. * in normalized timespec format in the variable pointed to by @ts.
  206. */
  207. void ktime_get_ts(struct timespec *ts)
  208. {
  209. cycle_t cycle_now, cycle_delta;
  210. struct clocksource *clock;
  211. struct timespec tomono;
  212. unsigned int seq;
  213. s64 nsecs;
  214. WARN_ON(timekeeping_suspended);
  215. do {
  216. seq = read_seqbegin(&xtime_lock);
  217. *ts = xtime;
  218. tomono = wall_to_monotonic;
  219. /* read clocksource: */
  220. clock = timekeeper.clock;
  221. cycle_now = clock->read(clock);
  222. /* calculate the delta since the last update_wall_time: */
  223. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  224. /* convert to nanoseconds: */
  225. nsecs = clocksource_cyc2ns(cycle_delta, clock->mult,
  226. clock->shift);
  227. } while (read_seqretry(&xtime_lock, seq));
  228. set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
  229. ts->tv_nsec + tomono.tv_nsec + nsecs);
  230. }
  231. EXPORT_SYMBOL_GPL(ktime_get_ts);
  232. /**
  233. * do_gettimeofday - Returns the time of day in a timeval
  234. * @tv: pointer to the timeval to be set
  235. *
  236. * NOTE: Users should be converted to using getnstimeofday()
  237. */
  238. void do_gettimeofday(struct timeval *tv)
  239. {
  240. struct timespec now;
  241. getnstimeofday(&now);
  242. tv->tv_sec = now.tv_sec;
  243. tv->tv_usec = now.tv_nsec/1000;
  244. }
  245. EXPORT_SYMBOL(do_gettimeofday);
  246. /**
  247. * do_settimeofday - Sets the time of day
  248. * @tv: pointer to the timespec variable containing the new time
  249. *
  250. * Sets the time of day to the new time and update NTP and notify hrtimers
  251. */
  252. int do_settimeofday(struct timespec *tv)
  253. {
  254. struct timespec ts_delta;
  255. unsigned long flags;
  256. if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
  257. return -EINVAL;
  258. write_seqlock_irqsave(&xtime_lock, flags);
  259. timekeeping_forward_now();
  260. ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec;
  261. ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec;
  262. wall_to_monotonic = timespec_sub(wall_to_monotonic, ts_delta);
  263. xtime = *tv;
  264. update_xtime_cache(0);
  265. timekeeper.ntp_error = 0;
  266. ntp_clear();
  267. update_vsyscall(&xtime, timekeeper.clock);
  268. write_sequnlock_irqrestore(&xtime_lock, flags);
  269. /* signal hrtimers about time change */
  270. clock_was_set();
  271. return 0;
  272. }
  273. EXPORT_SYMBOL(do_settimeofday);
  274. /**
  275. * change_clocksource - Swaps clocksources if a new one is available
  276. *
  277. * Accumulates current time interval and initializes new clocksource
  278. */
  279. static void change_clocksource(void)
  280. {
  281. struct clocksource *new, *old;
  282. new = clocksource_get_next();
  283. if (!new || timekeeper.clock == new)
  284. return;
  285. timekeeping_forward_now();
  286. if (new->enable && !new->enable(new))
  287. return;
  288. /*
  289. * The frequency may have changed while the clocksource
  290. * was disabled. If so the code in ->enable() must update
  291. * the mult value to reflect the new frequency. Make sure
  292. * mult_orig follows this change.
  293. */
  294. new->mult_orig = new->mult;
  295. old = timekeeper.clock;
  296. timekeeper_setup_internals(new);
  297. /*
  298. * Save mult_orig in mult so that the value can be restored
  299. * regardless if ->enable() updates the value of mult or not.
  300. */
  301. old->mult = old->mult_orig;
  302. if (old->disable)
  303. old->disable(old);
  304. tick_clock_notify();
  305. }
  306. #else /* GENERIC_TIME */
  307. static inline void timekeeping_forward_now(void) { }
  308. static inline void change_clocksource(void) { }
  309. /**
  310. * ktime_get - get the monotonic time in ktime_t format
  311. *
  312. * returns the time in ktime_t format
  313. */
  314. ktime_t ktime_get(void)
  315. {
  316. struct timespec now;
  317. ktime_get_ts(&now);
  318. return timespec_to_ktime(now);
  319. }
  320. EXPORT_SYMBOL_GPL(ktime_get);
  321. /**
  322. * ktime_get_ts - get the monotonic clock in timespec format
  323. * @ts: pointer to timespec variable
  324. *
  325. * The function calculates the monotonic clock from the realtime
  326. * clock and the wall_to_monotonic offset and stores the result
  327. * in normalized timespec format in the variable pointed to by @ts.
  328. */
  329. void ktime_get_ts(struct timespec *ts)
  330. {
  331. struct timespec tomono;
  332. unsigned long seq;
  333. do {
  334. seq = read_seqbegin(&xtime_lock);
  335. getnstimeofday(ts);
  336. tomono = wall_to_monotonic;
  337. } while (read_seqretry(&xtime_lock, seq));
  338. set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
  339. ts->tv_nsec + tomono.tv_nsec);
  340. }
  341. EXPORT_SYMBOL_GPL(ktime_get_ts);
  342. #endif /* !GENERIC_TIME */
  343. /**
  344. * ktime_get_real - get the real (wall-) time in ktime_t format
  345. *
  346. * returns the time in ktime_t format
  347. */
  348. ktime_t ktime_get_real(void)
  349. {
  350. struct timespec now;
  351. getnstimeofday(&now);
  352. return timespec_to_ktime(now);
  353. }
  354. EXPORT_SYMBOL_GPL(ktime_get_real);
  355. /**
  356. * getrawmonotonic - Returns the raw monotonic time in a timespec
  357. * @ts: pointer to the timespec to be set
  358. *
  359. * Returns the raw monotonic time (completely un-modified by ntp)
  360. */
  361. void getrawmonotonic(struct timespec *ts)
  362. {
  363. unsigned long seq;
  364. s64 nsecs;
  365. cycle_t cycle_now, cycle_delta;
  366. struct clocksource *clock;
  367. do {
  368. seq = read_seqbegin(&xtime_lock);
  369. /* read clocksource: */
  370. clock = timekeeper.clock;
  371. cycle_now = clock->read(clock);
  372. /* calculate the delta since the last update_wall_time: */
  373. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  374. /* convert to nanoseconds: */
  375. nsecs = clocksource_cyc2ns(cycle_delta, clock->mult_orig,
  376. clock->shift);
  377. *ts = raw_time;
  378. } while (read_seqretry(&xtime_lock, seq));
  379. timespec_add_ns(ts, nsecs);
  380. }
  381. EXPORT_SYMBOL(getrawmonotonic);
  382. /**
  383. * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
  384. */
  385. int timekeeping_valid_for_hres(void)
  386. {
  387. unsigned long seq;
  388. int ret;
  389. do {
  390. seq = read_seqbegin(&xtime_lock);
  391. ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
  392. } while (read_seqretry(&xtime_lock, seq));
  393. return ret;
  394. }
  395. /**
  396. * read_persistent_clock - Return time in seconds from the persistent clock.
  397. *
  398. * Weak dummy function for arches that do not yet support it.
  399. * Returns seconds from epoch using the battery backed persistent clock.
  400. * Returns zero if unsupported.
  401. *
  402. * XXX - Do be sure to remove it once all arches implement it.
  403. */
  404. unsigned long __attribute__((weak)) read_persistent_clock(void)
  405. {
  406. return 0;
  407. }
  408. /*
  409. * timekeeping_init - Initializes the clocksource and common timekeeping values
  410. */
  411. void __init timekeeping_init(void)
  412. {
  413. struct clocksource *clock;
  414. unsigned long flags;
  415. unsigned long sec = read_persistent_clock();
  416. write_seqlock_irqsave(&xtime_lock, flags);
  417. ntp_init();
  418. clock = clocksource_default_clock();
  419. if (clock->enable)
  420. clock->enable(clock);
  421. /* set mult_orig on enable */
  422. clock->mult_orig = clock->mult;
  423. timekeeper_setup_internals(clock);
  424. xtime.tv_sec = sec;
  425. xtime.tv_nsec = 0;
  426. raw_time.tv_sec = 0;
  427. raw_time.tv_nsec = 0;
  428. set_normalized_timespec(&wall_to_monotonic,
  429. -xtime.tv_sec, -xtime.tv_nsec);
  430. update_xtime_cache(0);
  431. total_sleep_time = 0;
  432. write_sequnlock_irqrestore(&xtime_lock, flags);
  433. }
  434. /* time in seconds when suspend began */
  435. static unsigned long timekeeping_suspend_time;
  436. /**
  437. * timekeeping_resume - Resumes the generic timekeeping subsystem.
  438. * @dev: unused
  439. *
  440. * This is for the generic clocksource timekeeping.
  441. * xtime/wall_to_monotonic/jiffies/etc are
  442. * still managed by arch specific suspend/resume code.
  443. */
  444. static int timekeeping_resume(struct sys_device *dev)
  445. {
  446. unsigned long flags;
  447. unsigned long now = read_persistent_clock();
  448. clocksource_resume();
  449. write_seqlock_irqsave(&xtime_lock, flags);
  450. if (now && (now > timekeeping_suspend_time)) {
  451. unsigned long sleep_length = now - timekeeping_suspend_time;
  452. xtime.tv_sec += sleep_length;
  453. wall_to_monotonic.tv_sec -= sleep_length;
  454. total_sleep_time += sleep_length;
  455. }
  456. update_xtime_cache(0);
  457. /* re-base the last cycle value */
  458. timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
  459. timekeeper.ntp_error = 0;
  460. timekeeping_suspended = 0;
  461. write_sequnlock_irqrestore(&xtime_lock, flags);
  462. touch_softlockup_watchdog();
  463. clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
  464. /* Resume hrtimers */
  465. hres_timers_resume();
  466. return 0;
  467. }
  468. static int timekeeping_suspend(struct sys_device *dev, pm_message_t state)
  469. {
  470. unsigned long flags;
  471. timekeeping_suspend_time = read_persistent_clock();
  472. write_seqlock_irqsave(&xtime_lock, flags);
  473. timekeeping_forward_now();
  474. timekeeping_suspended = 1;
  475. write_sequnlock_irqrestore(&xtime_lock, flags);
  476. clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
  477. return 0;
  478. }
  479. /* sysfs resume/suspend bits for timekeeping */
  480. static struct sysdev_class timekeeping_sysclass = {
  481. .name = "timekeeping",
  482. .resume = timekeeping_resume,
  483. .suspend = timekeeping_suspend,
  484. };
  485. static struct sys_device device_timer = {
  486. .id = 0,
  487. .cls = &timekeeping_sysclass,
  488. };
  489. static int __init timekeeping_init_device(void)
  490. {
  491. int error = sysdev_class_register(&timekeeping_sysclass);
  492. if (!error)
  493. error = sysdev_register(&device_timer);
  494. return error;
  495. }
  496. device_initcall(timekeeping_init_device);
  497. /*
  498. * If the error is already larger, we look ahead even further
  499. * to compensate for late or lost adjustments.
  500. */
  501. static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
  502. s64 *offset)
  503. {
  504. s64 tick_error, i;
  505. u32 look_ahead, adj;
  506. s32 error2, mult;
  507. /*
  508. * Use the current error value to determine how much to look ahead.
  509. * The larger the error the slower we adjust for it to avoid problems
  510. * with losing too many ticks, otherwise we would overadjust and
  511. * produce an even larger error. The smaller the adjustment the
  512. * faster we try to adjust for it, as lost ticks can do less harm
  513. * here. This is tuned so that an error of about 1 msec is adjusted
  514. * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
  515. */
  516. error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
  517. error2 = abs(error2);
  518. for (look_ahead = 0; error2 > 0; look_ahead++)
  519. error2 >>= 2;
  520. /*
  521. * Now calculate the error in (1 << look_ahead) ticks, but first
  522. * remove the single look ahead already included in the error.
  523. */
  524. tick_error = tick_length >> (timekeeper.ntp_error_shift + 1);
  525. tick_error -= timekeeper.xtime_interval >> 1;
  526. error = ((error - tick_error) >> look_ahead) + tick_error;
  527. /* Finally calculate the adjustment shift value. */
  528. i = *interval;
  529. mult = 1;
  530. if (error < 0) {
  531. error = -error;
  532. *interval = -*interval;
  533. *offset = -*offset;
  534. mult = -1;
  535. }
  536. for (adj = 0; error > i; adj++)
  537. error >>= 1;
  538. *interval <<= adj;
  539. *offset <<= adj;
  540. return mult << adj;
  541. }
  542. /*
  543. * Adjust the multiplier to reduce the error value,
  544. * this is optimized for the most common adjustments of -1,0,1,
  545. * for other values we can do a bit more work.
  546. */
  547. static void timekeeping_adjust(s64 offset)
  548. {
  549. s64 error, interval = timekeeper.cycle_interval;
  550. int adj;
  551. error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1);
  552. if (error > interval) {
  553. error >>= 2;
  554. if (likely(error <= interval))
  555. adj = 1;
  556. else
  557. adj = timekeeping_bigadjust(error, &interval, &offset);
  558. } else if (error < -interval) {
  559. error >>= 2;
  560. if (likely(error >= -interval)) {
  561. adj = -1;
  562. interval = -interval;
  563. offset = -offset;
  564. } else
  565. adj = timekeeping_bigadjust(error, &interval, &offset);
  566. } else
  567. return;
  568. timekeeper.clock->mult += adj;
  569. timekeeper.xtime_interval += interval;
  570. timekeeper.xtime_nsec -= offset;
  571. timekeeper.ntp_error -= (interval - offset) <<
  572. timekeeper.ntp_error_shift;
  573. }
  574. /**
  575. * update_wall_time - Uses the current clocksource to increment the wall time
  576. *
  577. * Called from the timer interrupt, must hold a write on xtime_lock.
  578. */
  579. void update_wall_time(void)
  580. {
  581. struct clocksource *clock;
  582. cycle_t offset;
  583. u64 nsecs;
  584. /* Make sure we're fully resumed: */
  585. if (unlikely(timekeeping_suspended))
  586. return;
  587. clock = timekeeper.clock;
  588. #ifdef CONFIG_GENERIC_TIME
  589. offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
  590. #else
  591. offset = timekeeper.cycle_interval;
  592. #endif
  593. timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift;
  594. /* normally this loop will run just once, however in the
  595. * case of lost or late ticks, it will accumulate correctly.
  596. */
  597. while (offset >= timekeeper.cycle_interval) {
  598. u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift;
  599. /* accumulate one interval */
  600. offset -= timekeeper.cycle_interval;
  601. clock->cycle_last += timekeeper.cycle_interval;
  602. timekeeper.xtime_nsec += timekeeper.xtime_interval;
  603. if (timekeeper.xtime_nsec >= nsecps) {
  604. timekeeper.xtime_nsec -= nsecps;
  605. xtime.tv_sec++;
  606. second_overflow();
  607. }
  608. raw_time.tv_nsec += timekeeper.raw_interval;
  609. if (raw_time.tv_nsec >= NSEC_PER_SEC) {
  610. raw_time.tv_nsec -= NSEC_PER_SEC;
  611. raw_time.tv_sec++;
  612. }
  613. /* accumulate error between NTP and clock interval */
  614. timekeeper.ntp_error += tick_length;
  615. timekeeper.ntp_error -= timekeeper.xtime_interval <<
  616. timekeeper.ntp_error_shift;
  617. }
  618. /* correct the clock when NTP error is too big */
  619. timekeeping_adjust(offset);
  620. /*
  621. * Since in the loop above, we accumulate any amount of time
  622. * in xtime_nsec over a second into xtime.tv_sec, its possible for
  623. * xtime_nsec to be fairly small after the loop. Further, if we're
  624. * slightly speeding the clocksource up in timekeeping_adjust(),
  625. * its possible the required corrective factor to xtime_nsec could
  626. * cause it to underflow.
  627. *
  628. * Now, we cannot simply roll the accumulated second back, since
  629. * the NTP subsystem has been notified via second_overflow. So
  630. * instead we push xtime_nsec forward by the amount we underflowed,
  631. * and add that amount into the error.
  632. *
  633. * We'll correct this error next time through this function, when
  634. * xtime_nsec is not as small.
  635. */
  636. if (unlikely((s64)timekeeper.xtime_nsec < 0)) {
  637. s64 neg = -(s64)timekeeper.xtime_nsec;
  638. timekeeper.xtime_nsec = 0;
  639. timekeeper.ntp_error += neg << timekeeper.ntp_error_shift;
  640. }
  641. /* store full nanoseconds into xtime after rounding it up and
  642. * add the remainder to the error difference.
  643. */
  644. xtime.tv_nsec = ((s64) timekeeper.xtime_nsec >> timekeeper.shift) + 1;
  645. timekeeper.xtime_nsec -= (s64) xtime.tv_nsec << timekeeper.shift;
  646. timekeeper.ntp_error += timekeeper.xtime_nsec <<
  647. timekeeper.ntp_error_shift;
  648. nsecs = clocksource_cyc2ns(offset, clock->mult, clock->shift);
  649. update_xtime_cache(nsecs);
  650. /* check to see if there is a new clocksource to use */
  651. change_clocksource();
  652. update_vsyscall(&xtime, timekeeper.clock);
  653. }
  654. /**
  655. * getboottime - Return the real time of system boot.
  656. * @ts: pointer to the timespec to be set
  657. *
  658. * Returns the time of day in a timespec.
  659. *
  660. * This is based on the wall_to_monotonic offset and the total suspend
  661. * time. Calls to settimeofday will affect the value returned (which
  662. * basically means that however wrong your real time clock is at boot time,
  663. * you get the right time here).
  664. */
  665. void getboottime(struct timespec *ts)
  666. {
  667. set_normalized_timespec(ts,
  668. - (wall_to_monotonic.tv_sec + total_sleep_time),
  669. - wall_to_monotonic.tv_nsec);
  670. }
  671. /**
  672. * monotonic_to_bootbased - Convert the monotonic time to boot based.
  673. * @ts: pointer to the timespec to be converted
  674. */
  675. void monotonic_to_bootbased(struct timespec *ts)
  676. {
  677. ts->tv_sec += total_sleep_time;
  678. }
  679. unsigned long get_seconds(void)
  680. {
  681. return xtime_cache.tv_sec;
  682. }
  683. EXPORT_SYMBOL(get_seconds);
  684. struct timespec current_kernel_time(void)
  685. {
  686. struct timespec now;
  687. unsigned long seq;
  688. do {
  689. seq = read_seqbegin(&xtime_lock);
  690. now = xtime_cache;
  691. } while (read_seqretry(&xtime_lock, seq));
  692. return now;
  693. }
  694. EXPORT_SYMBOL(current_kernel_time);