timekeeping.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842
  1. /*
  2. * linux/kernel/time/timekeeping.c
  3. *
  4. * Kernel timekeeping code and accessor functions
  5. *
  6. * This code was moved from linux/kernel/timer.c.
  7. * Please see that file for copyright and history logs.
  8. *
  9. */
  10. #include <linux/module.h>
  11. #include <linux/interrupt.h>
  12. #include <linux/percpu.h>
  13. #include <linux/init.h>
  14. #include <linux/mm.h>
  15. #include <linux/sysdev.h>
  16. #include <linux/clocksource.h>
  17. #include <linux/jiffies.h>
  18. #include <linux/time.h>
  19. #include <linux/tick.h>
  20. /* Structure holding internal timekeeping values. */
  21. struct timekeeper {
  22. /* Current clocksource used for timekeeping. */
  23. struct clocksource *clock;
  24. /* The shift value of the current clocksource. */
  25. int shift;
  26. /* Number of clock cycles in one NTP interval. */
  27. cycle_t cycle_interval;
  28. /* Number of clock shifted nano seconds in one NTP interval. */
  29. u64 xtime_interval;
  30. /* Raw nano seconds accumulated per NTP interval. */
  31. u32 raw_interval;
  32. /* Clock shifted nano seconds remainder not stored in xtime.tv_nsec. */
  33. u64 xtime_nsec;
  34. /* Difference between accumulated time and NTP time in ntp
  35. * shifted nano seconds. */
  36. s64 ntp_error;
  37. /* Shift conversion between clock shifted nano seconds and
  38. * ntp shifted nano seconds. */
  39. int ntp_error_shift;
  40. /* NTP adjusted clock multiplier */
  41. u32 mult;
  42. };
  43. struct timekeeper timekeeper;
  44. /**
  45. * timekeeper_setup_internals - Set up internals to use clocksource clock.
  46. *
  47. * @clock: Pointer to clocksource.
  48. *
  49. * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
  50. * pair and interval request.
  51. *
  52. * Unless you're the timekeeping code, you should not be using this!
  53. */
  54. static void timekeeper_setup_internals(struct clocksource *clock)
  55. {
  56. cycle_t interval;
  57. u64 tmp;
  58. timekeeper.clock = clock;
  59. clock->cycle_last = clock->read(clock);
  60. /* Do the ns -> cycle conversion first, using original mult */
  61. tmp = NTP_INTERVAL_LENGTH;
  62. tmp <<= clock->shift;
  63. tmp += clock->mult/2;
  64. do_div(tmp, clock->mult);
  65. if (tmp == 0)
  66. tmp = 1;
  67. interval = (cycle_t) tmp;
  68. timekeeper.cycle_interval = interval;
  69. /* Go back from cycles -> shifted ns */
  70. timekeeper.xtime_interval = (u64) interval * clock->mult;
  71. timekeeper.raw_interval =
  72. ((u64) interval * clock->mult) >> clock->shift;
  73. timekeeper.xtime_nsec = 0;
  74. timekeeper.shift = clock->shift;
  75. timekeeper.ntp_error = 0;
  76. timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
  77. /*
  78. * The timekeeper keeps its own mult values for the currently
  79. * active clocksource. These value will be adjusted via NTP
  80. * to counteract clock drifting.
  81. */
  82. timekeeper.mult = clock->mult;
  83. }
  84. /*
  85. * This read-write spinlock protects us from races in SMP while
  86. * playing with xtime.
  87. */
  88. __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);
  89. /*
  90. * The current time
  91. * wall_to_monotonic is what we need to add to xtime (or xtime corrected
  92. * for sub jiffie times) to get to monotonic time. Monotonic is pegged
  93. * at zero at system boot time, so wall_to_monotonic will be negative,
  94. * however, we will ALWAYS keep the tv_nsec part positive so we can use
  95. * the usual normalization.
  96. *
  97. * wall_to_monotonic is moved after resume from suspend for the monotonic
  98. * time not to jump. We need to add total_sleep_time to wall_to_monotonic
  99. * to get the real boot based time offset.
  100. *
  101. * - wall_to_monotonic is no longer the boot time, getboottime must be
  102. * used instead.
  103. */
  104. struct timespec xtime __attribute__ ((aligned (16)));
  105. struct timespec wall_to_monotonic __attribute__ ((aligned (16)));
  106. static unsigned long total_sleep_time; /* seconds */
  107. /*
  108. * The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock.
  109. */
  110. struct timespec raw_time;
  111. /* flag for if timekeeping is suspended */
  112. int __read_mostly timekeeping_suspended;
  113. static struct timespec xtime_cache __attribute__ ((aligned (16)));
  114. void update_xtime_cache(u64 nsec)
  115. {
  116. xtime_cache = xtime;
  117. timespec_add_ns(&xtime_cache, nsec);
  118. }
  119. /* must hold xtime_lock */
  120. void timekeeping_leap_insert(int leapsecond)
  121. {
  122. xtime.tv_sec += leapsecond;
  123. wall_to_monotonic.tv_sec -= leapsecond;
  124. update_vsyscall(&xtime, timekeeper.clock);
  125. }
  126. #ifdef CONFIG_GENERIC_TIME
  127. /**
  128. * timekeeping_forward_now - update clock to the current time
  129. *
  130. * Forward the current clock to update its state since the last call to
  131. * update_wall_time(). This is useful before significant clock changes,
  132. * as it avoids having to deal with this time offset explicitly.
  133. */
  134. static void timekeeping_forward_now(void)
  135. {
  136. cycle_t cycle_now, cycle_delta;
  137. struct clocksource *clock;
  138. s64 nsec;
  139. clock = timekeeper.clock;
  140. cycle_now = clock->read(clock);
  141. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  142. clock->cycle_last = cycle_now;
  143. nsec = clocksource_cyc2ns(cycle_delta, timekeeper.mult,
  144. timekeeper.shift);
  145. /* If arch requires, add in gettimeoffset() */
  146. nsec += arch_gettimeoffset();
  147. timespec_add_ns(&xtime, nsec);
  148. nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
  149. timespec_add_ns(&raw_time, nsec);
  150. }
  151. /**
  152. * getnstimeofday - Returns the time of day in a timespec
  153. * @ts: pointer to the timespec to be set
  154. *
  155. * Returns the time of day in a timespec.
  156. */
  157. void getnstimeofday(struct timespec *ts)
  158. {
  159. cycle_t cycle_now, cycle_delta;
  160. struct clocksource *clock;
  161. unsigned long seq;
  162. s64 nsecs;
  163. WARN_ON(timekeeping_suspended);
  164. do {
  165. seq = read_seqbegin(&xtime_lock);
  166. *ts = xtime;
  167. /* read clocksource: */
  168. clock = timekeeper.clock;
  169. cycle_now = clock->read(clock);
  170. /* calculate the delta since the last update_wall_time: */
  171. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  172. /* convert to nanoseconds: */
  173. nsecs = clocksource_cyc2ns(cycle_delta, timekeeper.mult,
  174. timekeeper.shift);
  175. /* If arch requires, add in gettimeoffset() */
  176. nsecs += arch_gettimeoffset();
  177. } while (read_seqretry(&xtime_lock, seq));
  178. timespec_add_ns(ts, nsecs);
  179. }
  180. EXPORT_SYMBOL(getnstimeofday);
  181. ktime_t ktime_get(void)
  182. {
  183. cycle_t cycle_now, cycle_delta;
  184. struct clocksource *clock;
  185. unsigned int seq;
  186. s64 secs, nsecs;
  187. WARN_ON(timekeeping_suspended);
  188. do {
  189. seq = read_seqbegin(&xtime_lock);
  190. secs = xtime.tv_sec + wall_to_monotonic.tv_sec;
  191. nsecs = xtime.tv_nsec + wall_to_monotonic.tv_nsec;
  192. /* read clocksource: */
  193. clock = timekeeper.clock;
  194. cycle_now = clock->read(clock);
  195. /* calculate the delta since the last update_wall_time: */
  196. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  197. /* convert to nanoseconds: */
  198. nsecs += clocksource_cyc2ns(cycle_delta, timekeeper.mult,
  199. timekeeper.shift);
  200. } while (read_seqretry(&xtime_lock, seq));
  201. /*
  202. * Use ktime_set/ktime_add_ns to create a proper ktime on
  203. * 32-bit architectures without CONFIG_KTIME_SCALAR.
  204. */
  205. return ktime_add_ns(ktime_set(secs, 0), nsecs);
  206. }
  207. EXPORT_SYMBOL_GPL(ktime_get);
  208. /**
  209. * ktime_get_ts - get the monotonic clock in timespec format
  210. * @ts: pointer to timespec variable
  211. *
  212. * The function calculates the monotonic clock from the realtime
  213. * clock and the wall_to_monotonic offset and stores the result
  214. * in normalized timespec format in the variable pointed to by @ts.
  215. */
  216. void ktime_get_ts(struct timespec *ts)
  217. {
  218. cycle_t cycle_now, cycle_delta;
  219. struct clocksource *clock;
  220. struct timespec tomono;
  221. unsigned int seq;
  222. s64 nsecs;
  223. WARN_ON(timekeeping_suspended);
  224. do {
  225. seq = read_seqbegin(&xtime_lock);
  226. *ts = xtime;
  227. tomono = wall_to_monotonic;
  228. /* read clocksource: */
  229. clock = timekeeper.clock;
  230. cycle_now = clock->read(clock);
  231. /* calculate the delta since the last update_wall_time: */
  232. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  233. /* convert to nanoseconds: */
  234. nsecs = clocksource_cyc2ns(cycle_delta, timekeeper.mult,
  235. timekeeper.shift);
  236. } while (read_seqretry(&xtime_lock, seq));
  237. set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
  238. ts->tv_nsec + tomono.tv_nsec + nsecs);
  239. }
  240. EXPORT_SYMBOL_GPL(ktime_get_ts);
  241. /**
  242. * do_gettimeofday - Returns the time of day in a timeval
  243. * @tv: pointer to the timeval to be set
  244. *
  245. * NOTE: Users should be converted to using getnstimeofday()
  246. */
  247. void do_gettimeofday(struct timeval *tv)
  248. {
  249. struct timespec now;
  250. getnstimeofday(&now);
  251. tv->tv_sec = now.tv_sec;
  252. tv->tv_usec = now.tv_nsec/1000;
  253. }
  254. EXPORT_SYMBOL(do_gettimeofday);
  255. /**
  256. * do_settimeofday - Sets the time of day
  257. * @tv: pointer to the timespec variable containing the new time
  258. *
  259. * Sets the time of day to the new time and update NTP and notify hrtimers
  260. */
  261. int do_settimeofday(struct timespec *tv)
  262. {
  263. struct timespec ts_delta;
  264. unsigned long flags;
  265. if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
  266. return -EINVAL;
  267. write_seqlock_irqsave(&xtime_lock, flags);
  268. timekeeping_forward_now();
  269. ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec;
  270. ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec;
  271. wall_to_monotonic = timespec_sub(wall_to_monotonic, ts_delta);
  272. xtime = *tv;
  273. update_xtime_cache(0);
  274. timekeeper.ntp_error = 0;
  275. ntp_clear();
  276. update_vsyscall(&xtime, timekeeper.clock);
  277. write_sequnlock_irqrestore(&xtime_lock, flags);
  278. /* signal hrtimers about time change */
  279. clock_was_set();
  280. return 0;
  281. }
  282. EXPORT_SYMBOL(do_settimeofday);
  283. /**
  284. * change_clocksource - Swaps clocksources if a new one is available
  285. *
  286. * Accumulates current time interval and initializes new clocksource
  287. */
  288. static void change_clocksource(void)
  289. {
  290. struct clocksource *new, *old;
  291. new = clocksource_get_next();
  292. if (!new || timekeeper.clock == new)
  293. return;
  294. timekeeping_forward_now();
  295. if (new->enable && !new->enable(new))
  296. return;
  297. old = timekeeper.clock;
  298. timekeeper_setup_internals(new);
  299. if (old->disable)
  300. old->disable(old);
  301. tick_clock_notify();
  302. }
  303. #else /* GENERIC_TIME */
  304. static inline void timekeeping_forward_now(void) { }
  305. static inline void change_clocksource(void) { }
  306. /**
  307. * ktime_get - get the monotonic time in ktime_t format
  308. *
  309. * returns the time in ktime_t format
  310. */
  311. ktime_t ktime_get(void)
  312. {
  313. struct timespec now;
  314. ktime_get_ts(&now);
  315. return timespec_to_ktime(now);
  316. }
  317. EXPORT_SYMBOL_GPL(ktime_get);
  318. /**
  319. * ktime_get_ts - get the monotonic clock in timespec format
  320. * @ts: pointer to timespec variable
  321. *
  322. * The function calculates the monotonic clock from the realtime
  323. * clock and the wall_to_monotonic offset and stores the result
  324. * in normalized timespec format in the variable pointed to by @ts.
  325. */
  326. void ktime_get_ts(struct timespec *ts)
  327. {
  328. struct timespec tomono;
  329. unsigned long seq;
  330. do {
  331. seq = read_seqbegin(&xtime_lock);
  332. getnstimeofday(ts);
  333. tomono = wall_to_monotonic;
  334. } while (read_seqretry(&xtime_lock, seq));
  335. set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
  336. ts->tv_nsec + tomono.tv_nsec);
  337. }
  338. EXPORT_SYMBOL_GPL(ktime_get_ts);
  339. #endif /* !GENERIC_TIME */
  340. /**
  341. * ktime_get_real - get the real (wall-) time in ktime_t format
  342. *
  343. * returns the time in ktime_t format
  344. */
  345. ktime_t ktime_get_real(void)
  346. {
  347. struct timespec now;
  348. getnstimeofday(&now);
  349. return timespec_to_ktime(now);
  350. }
  351. EXPORT_SYMBOL_GPL(ktime_get_real);
  352. /**
  353. * getrawmonotonic - Returns the raw monotonic time in a timespec
  354. * @ts: pointer to the timespec to be set
  355. *
  356. * Returns the raw monotonic time (completely un-modified by ntp)
  357. */
  358. void getrawmonotonic(struct timespec *ts)
  359. {
  360. unsigned long seq;
  361. s64 nsecs;
  362. cycle_t cycle_now, cycle_delta;
  363. struct clocksource *clock;
  364. do {
  365. seq = read_seqbegin(&xtime_lock);
  366. /* read clocksource: */
  367. clock = timekeeper.clock;
  368. cycle_now = clock->read(clock);
  369. /* calculate the delta since the last update_wall_time: */
  370. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  371. /* convert to nanoseconds: */
  372. nsecs = clocksource_cyc2ns(cycle_delta, clock->mult,
  373. clock->shift);
  374. *ts = raw_time;
  375. } while (read_seqretry(&xtime_lock, seq));
  376. timespec_add_ns(ts, nsecs);
  377. }
  378. EXPORT_SYMBOL(getrawmonotonic);
  379. /**
  380. * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
  381. */
  382. int timekeeping_valid_for_hres(void)
  383. {
  384. unsigned long seq;
  385. int ret;
  386. do {
  387. seq = read_seqbegin(&xtime_lock);
  388. ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
  389. } while (read_seqretry(&xtime_lock, seq));
  390. return ret;
  391. }
  392. /**
  393. * read_persistent_clock - Return time in seconds from the persistent clock.
  394. *
  395. * Weak dummy function for arches that do not yet support it.
  396. * Returns seconds from epoch using the battery backed persistent clock.
  397. * Returns zero if unsupported.
  398. *
  399. * XXX - Do be sure to remove it once all arches implement it.
  400. */
  401. unsigned long __attribute__((weak)) read_persistent_clock(void)
  402. {
  403. return 0;
  404. }
  405. /*
  406. * timekeeping_init - Initializes the clocksource and common timekeeping values
  407. */
  408. void __init timekeeping_init(void)
  409. {
  410. struct clocksource *clock;
  411. unsigned long flags;
  412. unsigned long sec = read_persistent_clock();
  413. write_seqlock_irqsave(&xtime_lock, flags);
  414. ntp_init();
  415. clock = clocksource_default_clock();
  416. if (clock->enable)
  417. clock->enable(clock);
  418. timekeeper_setup_internals(clock);
  419. xtime.tv_sec = sec;
  420. xtime.tv_nsec = 0;
  421. raw_time.tv_sec = 0;
  422. raw_time.tv_nsec = 0;
  423. set_normalized_timespec(&wall_to_monotonic,
  424. -xtime.tv_sec, -xtime.tv_nsec);
  425. update_xtime_cache(0);
  426. total_sleep_time = 0;
  427. write_sequnlock_irqrestore(&xtime_lock, flags);
  428. }
  429. /* time in seconds when suspend began */
  430. static unsigned long timekeeping_suspend_time;
  431. /**
  432. * timekeeping_resume - Resumes the generic timekeeping subsystem.
  433. * @dev: unused
  434. *
  435. * This is for the generic clocksource timekeeping.
  436. * xtime/wall_to_monotonic/jiffies/etc are
  437. * still managed by arch specific suspend/resume code.
  438. */
  439. static int timekeeping_resume(struct sys_device *dev)
  440. {
  441. unsigned long flags;
  442. unsigned long now = read_persistent_clock();
  443. clocksource_resume();
  444. write_seqlock_irqsave(&xtime_lock, flags);
  445. if (now && (now > timekeeping_suspend_time)) {
  446. unsigned long sleep_length = now - timekeeping_suspend_time;
  447. xtime.tv_sec += sleep_length;
  448. wall_to_monotonic.tv_sec -= sleep_length;
  449. total_sleep_time += sleep_length;
  450. }
  451. update_xtime_cache(0);
  452. /* re-base the last cycle value */
  453. timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
  454. timekeeper.ntp_error = 0;
  455. timekeeping_suspended = 0;
  456. write_sequnlock_irqrestore(&xtime_lock, flags);
  457. touch_softlockup_watchdog();
  458. clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
  459. /* Resume hrtimers */
  460. hres_timers_resume();
  461. return 0;
  462. }
  463. static int timekeeping_suspend(struct sys_device *dev, pm_message_t state)
  464. {
  465. unsigned long flags;
  466. timekeeping_suspend_time = read_persistent_clock();
  467. write_seqlock_irqsave(&xtime_lock, flags);
  468. timekeeping_forward_now();
  469. timekeeping_suspended = 1;
  470. write_sequnlock_irqrestore(&xtime_lock, flags);
  471. clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
  472. return 0;
  473. }
  474. /* sysfs resume/suspend bits for timekeeping */
  475. static struct sysdev_class timekeeping_sysclass = {
  476. .name = "timekeeping",
  477. .resume = timekeeping_resume,
  478. .suspend = timekeeping_suspend,
  479. };
  480. static struct sys_device device_timer = {
  481. .id = 0,
  482. .cls = &timekeeping_sysclass,
  483. };
  484. static int __init timekeeping_init_device(void)
  485. {
  486. int error = sysdev_class_register(&timekeeping_sysclass);
  487. if (!error)
  488. error = sysdev_register(&device_timer);
  489. return error;
  490. }
  491. device_initcall(timekeeping_init_device);
  492. /*
  493. * If the error is already larger, we look ahead even further
  494. * to compensate for late or lost adjustments.
  495. */
  496. static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
  497. s64 *offset)
  498. {
  499. s64 tick_error, i;
  500. u32 look_ahead, adj;
  501. s32 error2, mult;
  502. /*
  503. * Use the current error value to determine how much to look ahead.
  504. * The larger the error the slower we adjust for it to avoid problems
  505. * with losing too many ticks, otherwise we would overadjust and
  506. * produce an even larger error. The smaller the adjustment the
  507. * faster we try to adjust for it, as lost ticks can do less harm
  508. * here. This is tuned so that an error of about 1 msec is adjusted
  509. * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
  510. */
  511. error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
  512. error2 = abs(error2);
  513. for (look_ahead = 0; error2 > 0; look_ahead++)
  514. error2 >>= 2;
  515. /*
  516. * Now calculate the error in (1 << look_ahead) ticks, but first
  517. * remove the single look ahead already included in the error.
  518. */
  519. tick_error = tick_length >> (timekeeper.ntp_error_shift + 1);
  520. tick_error -= timekeeper.xtime_interval >> 1;
  521. error = ((error - tick_error) >> look_ahead) + tick_error;
  522. /* Finally calculate the adjustment shift value. */
  523. i = *interval;
  524. mult = 1;
  525. if (error < 0) {
  526. error = -error;
  527. *interval = -*interval;
  528. *offset = -*offset;
  529. mult = -1;
  530. }
  531. for (adj = 0; error > i; adj++)
  532. error >>= 1;
  533. *interval <<= adj;
  534. *offset <<= adj;
  535. return mult << adj;
  536. }
  537. /*
  538. * Adjust the multiplier to reduce the error value,
  539. * this is optimized for the most common adjustments of -1,0,1,
  540. * for other values we can do a bit more work.
  541. */
  542. static void timekeeping_adjust(s64 offset)
  543. {
  544. s64 error, interval = timekeeper.cycle_interval;
  545. int adj;
  546. error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1);
  547. if (error > interval) {
  548. error >>= 2;
  549. if (likely(error <= interval))
  550. adj = 1;
  551. else
  552. adj = timekeeping_bigadjust(error, &interval, &offset);
  553. } else if (error < -interval) {
  554. error >>= 2;
  555. if (likely(error >= -interval)) {
  556. adj = -1;
  557. interval = -interval;
  558. offset = -offset;
  559. } else
  560. adj = timekeeping_bigadjust(error, &interval, &offset);
  561. } else
  562. return;
  563. timekeeper.mult += adj;
  564. timekeeper.xtime_interval += interval;
  565. timekeeper.xtime_nsec -= offset;
  566. timekeeper.ntp_error -= (interval - offset) <<
  567. timekeeper.ntp_error_shift;
  568. }
  569. /**
  570. * update_wall_time - Uses the current clocksource to increment the wall time
  571. *
  572. * Called from the timer interrupt, must hold a write on xtime_lock.
  573. */
  574. void update_wall_time(void)
  575. {
  576. struct clocksource *clock;
  577. cycle_t offset;
  578. u64 nsecs;
  579. /* Make sure we're fully resumed: */
  580. if (unlikely(timekeeping_suspended))
  581. return;
  582. clock = timekeeper.clock;
  583. #ifdef CONFIG_GENERIC_TIME
  584. offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
  585. #else
  586. offset = timekeeper.cycle_interval;
  587. #endif
  588. timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift;
  589. /* normally this loop will run just once, however in the
  590. * case of lost or late ticks, it will accumulate correctly.
  591. */
  592. while (offset >= timekeeper.cycle_interval) {
  593. u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift;
  594. /* accumulate one interval */
  595. offset -= timekeeper.cycle_interval;
  596. clock->cycle_last += timekeeper.cycle_interval;
  597. timekeeper.xtime_nsec += timekeeper.xtime_interval;
  598. if (timekeeper.xtime_nsec >= nsecps) {
  599. timekeeper.xtime_nsec -= nsecps;
  600. xtime.tv_sec++;
  601. second_overflow();
  602. }
  603. raw_time.tv_nsec += timekeeper.raw_interval;
  604. if (raw_time.tv_nsec >= NSEC_PER_SEC) {
  605. raw_time.tv_nsec -= NSEC_PER_SEC;
  606. raw_time.tv_sec++;
  607. }
  608. /* accumulate error between NTP and clock interval */
  609. timekeeper.ntp_error += tick_length;
  610. timekeeper.ntp_error -= timekeeper.xtime_interval <<
  611. timekeeper.ntp_error_shift;
  612. }
  613. /* correct the clock when NTP error is too big */
  614. timekeeping_adjust(offset);
  615. /*
  616. * Since in the loop above, we accumulate any amount of time
  617. * in xtime_nsec over a second into xtime.tv_sec, its possible for
  618. * xtime_nsec to be fairly small after the loop. Further, if we're
  619. * slightly speeding the clocksource up in timekeeping_adjust(),
  620. * its possible the required corrective factor to xtime_nsec could
  621. * cause it to underflow.
  622. *
  623. * Now, we cannot simply roll the accumulated second back, since
  624. * the NTP subsystem has been notified via second_overflow. So
  625. * instead we push xtime_nsec forward by the amount we underflowed,
  626. * and add that amount into the error.
  627. *
  628. * We'll correct this error next time through this function, when
  629. * xtime_nsec is not as small.
  630. */
  631. if (unlikely((s64)timekeeper.xtime_nsec < 0)) {
  632. s64 neg = -(s64)timekeeper.xtime_nsec;
  633. timekeeper.xtime_nsec = 0;
  634. timekeeper.ntp_error += neg << timekeeper.ntp_error_shift;
  635. }
  636. /* store full nanoseconds into xtime after rounding it up and
  637. * add the remainder to the error difference.
  638. */
  639. xtime.tv_nsec = ((s64) timekeeper.xtime_nsec >> timekeeper.shift) + 1;
  640. timekeeper.xtime_nsec -= (s64) xtime.tv_nsec << timekeeper.shift;
  641. timekeeper.ntp_error += timekeeper.xtime_nsec <<
  642. timekeeper.ntp_error_shift;
  643. nsecs = clocksource_cyc2ns(offset, timekeeper.mult, timekeeper.shift);
  644. update_xtime_cache(nsecs);
  645. /* check to see if there is a new clocksource to use */
  646. change_clocksource();
  647. update_vsyscall(&xtime, timekeeper.clock);
  648. }
  649. /**
  650. * getboottime - Return the real time of system boot.
  651. * @ts: pointer to the timespec to be set
  652. *
  653. * Returns the time of day in a timespec.
  654. *
  655. * This is based on the wall_to_monotonic offset and the total suspend
  656. * time. Calls to settimeofday will affect the value returned (which
  657. * basically means that however wrong your real time clock is at boot time,
  658. * you get the right time here).
  659. */
  660. void getboottime(struct timespec *ts)
  661. {
  662. set_normalized_timespec(ts,
  663. - (wall_to_monotonic.tv_sec + total_sleep_time),
  664. - wall_to_monotonic.tv_nsec);
  665. }
  666. /**
  667. * monotonic_to_bootbased - Convert the monotonic time to boot based.
  668. * @ts: pointer to the timespec to be converted
  669. */
  670. void monotonic_to_bootbased(struct timespec *ts)
  671. {
  672. ts->tv_sec += total_sleep_time;
  673. }
  674. unsigned long get_seconds(void)
  675. {
  676. return xtime_cache.tv_sec;
  677. }
  678. EXPORT_SYMBOL(get_seconds);
  679. struct timespec current_kernel_time(void)
  680. {
  681. struct timespec now;
  682. unsigned long seq;
  683. do {
  684. seq = read_seqbegin(&xtime_lock);
  685. now = xtime_cache;
  686. } while (read_seqretry(&xtime_lock, seq));
  687. return now;
  688. }
  689. EXPORT_SYMBOL(current_kernel_time);