timekeeping.c 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614
  1. /*
  2. * linux/kernel/time/timekeeping.c
  3. *
  4. * Kernel timekeeping code and accessor functions
  5. *
  6. * This code was moved from linux/kernel/timer.c.
  7. * Please see that file for copyright and history logs.
  8. *
  9. */
  10. #include <linux/timekeeper_internal.h>
  11. #include <linux/module.h>
  12. #include <linux/interrupt.h>
  13. #include <linux/percpu.h>
  14. #include <linux/init.h>
  15. #include <linux/mm.h>
  16. #include <linux/sched.h>
  17. #include <linux/syscore_ops.h>
  18. #include <linux/clocksource.h>
  19. #include <linux/jiffies.h>
  20. #include <linux/time.h>
  21. #include <linux/tick.h>
  22. #include <linux/stop_machine.h>
  23. #include <linux/pvclock_gtod.h>
  24. #include "tick-internal.h"
  25. static struct timekeeper timekeeper;
  26. /* flag for if timekeeping is suspended */
  27. int __read_mostly timekeeping_suspended;
  28. /* Flag for if there is a persistent clock on this platform */
  29. bool __read_mostly persistent_clock_exist = false;
  30. static inline void tk_normalize_xtime(struct timekeeper *tk)
  31. {
  32. while (tk->xtime_nsec >= ((u64)NSEC_PER_SEC << tk->shift)) {
  33. tk->xtime_nsec -= (u64)NSEC_PER_SEC << tk->shift;
  34. tk->xtime_sec++;
  35. }
  36. }
  37. static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts)
  38. {
  39. tk->xtime_sec = ts->tv_sec;
  40. tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift;
  41. }
  42. static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts)
  43. {
  44. tk->xtime_sec += ts->tv_sec;
  45. tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift;
  46. tk_normalize_xtime(tk);
  47. }
  48. static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm)
  49. {
  50. struct timespec tmp;
  51. /*
  52. * Verify consistency of: offset_real = -wall_to_monotonic
  53. * before modifying anything
  54. */
  55. set_normalized_timespec(&tmp, -tk->wall_to_monotonic.tv_sec,
  56. -tk->wall_to_monotonic.tv_nsec);
  57. WARN_ON_ONCE(tk->offs_real.tv64 != timespec_to_ktime(tmp).tv64);
  58. tk->wall_to_monotonic = wtm;
  59. set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
  60. tk->offs_real = timespec_to_ktime(tmp);
  61. tk->offs_tai = ktime_sub(tk->offs_real, ktime_set(tk->tai_offset, 0));
  62. }
  63. static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t)
  64. {
  65. /* Verify consistency before modifying */
  66. WARN_ON_ONCE(tk->offs_boot.tv64 != timespec_to_ktime(tk->total_sleep_time).tv64);
  67. tk->total_sleep_time = t;
  68. tk->offs_boot = timespec_to_ktime(t);
  69. }
  70. /**
  71. * timekeeper_setup_internals - Set up internals to use clocksource clock.
  72. *
  73. * @clock: Pointer to clocksource.
  74. *
  75. * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
  76. * pair and interval request.
  77. *
  78. * Unless you're the timekeeping code, you should not be using this!
  79. */
  80. static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
  81. {
  82. cycle_t interval;
  83. u64 tmp, ntpinterval;
  84. struct clocksource *old_clock;
  85. old_clock = tk->clock;
  86. tk->clock = clock;
  87. clock->cycle_last = clock->read(clock);
  88. /* Do the ns -> cycle conversion first, using original mult */
  89. tmp = NTP_INTERVAL_LENGTH;
  90. tmp <<= clock->shift;
  91. ntpinterval = tmp;
  92. tmp += clock->mult/2;
  93. do_div(tmp, clock->mult);
  94. if (tmp == 0)
  95. tmp = 1;
  96. interval = (cycle_t) tmp;
  97. tk->cycle_interval = interval;
  98. /* Go back from cycles -> shifted ns */
  99. tk->xtime_interval = (u64) interval * clock->mult;
  100. tk->xtime_remainder = ntpinterval - tk->xtime_interval;
  101. tk->raw_interval =
  102. ((u64) interval * clock->mult) >> clock->shift;
  103. /* if changing clocks, convert xtime_nsec shift units */
  104. if (old_clock) {
  105. int shift_change = clock->shift - old_clock->shift;
  106. if (shift_change < 0)
  107. tk->xtime_nsec >>= -shift_change;
  108. else
  109. tk->xtime_nsec <<= shift_change;
  110. }
  111. tk->shift = clock->shift;
  112. tk->ntp_error = 0;
  113. tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
  114. /*
  115. * The timekeeper keeps its own mult values for the currently
  116. * active clocksource. These value will be adjusted via NTP
  117. * to counteract clock drifting.
  118. */
  119. tk->mult = clock->mult;
  120. }
  121. /* Timekeeper helper functions. */
  122. #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
  123. u32 (*arch_gettimeoffset)(void);
  124. u32 get_arch_timeoffset(void)
  125. {
  126. if (likely(arch_gettimeoffset))
  127. return arch_gettimeoffset();
  128. return 0;
  129. }
  130. #else
  131. static inline u32 get_arch_timeoffset(void) { return 0; }
  132. #endif
  133. static inline s64 timekeeping_get_ns(struct timekeeper *tk)
  134. {
  135. cycle_t cycle_now, cycle_delta;
  136. struct clocksource *clock;
  137. s64 nsec;
  138. /* read clocksource: */
  139. clock = tk->clock;
  140. cycle_now = clock->read(clock);
  141. /* calculate the delta since the last update_wall_time: */
  142. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  143. nsec = cycle_delta * tk->mult + tk->xtime_nsec;
  144. nsec >>= tk->shift;
  145. /* If arch requires, add in get_arch_timeoffset() */
  146. return nsec + get_arch_timeoffset();
  147. }
  148. static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
  149. {
  150. cycle_t cycle_now, cycle_delta;
  151. struct clocksource *clock;
  152. s64 nsec;
  153. /* read clocksource: */
  154. clock = tk->clock;
  155. cycle_now = clock->read(clock);
  156. /* calculate the delta since the last update_wall_time: */
  157. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  158. /* convert delta to nanoseconds. */
  159. nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
  160. /* If arch requires, add in get_arch_timeoffset() */
  161. return nsec + get_arch_timeoffset();
  162. }
  163. static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
  164. static void update_pvclock_gtod(struct timekeeper *tk)
  165. {
  166. raw_notifier_call_chain(&pvclock_gtod_chain, 0, tk);
  167. }
  168. /**
  169. * pvclock_gtod_register_notifier - register a pvclock timedata update listener
  170. *
  171. * Must hold write on timekeeper.lock
  172. */
  173. int pvclock_gtod_register_notifier(struct notifier_block *nb)
  174. {
  175. struct timekeeper *tk = &timekeeper;
  176. unsigned long flags;
  177. int ret;
  178. write_seqlock_irqsave(&tk->lock, flags);
  179. ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
  180. /* update timekeeping data */
  181. update_pvclock_gtod(tk);
  182. write_sequnlock_irqrestore(&tk->lock, flags);
  183. return ret;
  184. }
  185. EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
  186. /**
  187. * pvclock_gtod_unregister_notifier - unregister a pvclock
  188. * timedata update listener
  189. *
  190. * Must hold write on timekeeper.lock
  191. */
  192. int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
  193. {
  194. struct timekeeper *tk = &timekeeper;
  195. unsigned long flags;
  196. int ret;
  197. write_seqlock_irqsave(&tk->lock, flags);
  198. ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
  199. write_sequnlock_irqrestore(&tk->lock, flags);
  200. return ret;
  201. }
  202. EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
  203. /* must hold write on timekeeper.lock */
  204. static void timekeeping_update(struct timekeeper *tk, bool clearntp)
  205. {
  206. if (clearntp) {
  207. tk->ntp_error = 0;
  208. ntp_clear();
  209. }
  210. update_vsyscall(tk);
  211. update_pvclock_gtod(tk);
  212. }
  213. /**
  214. * timekeeping_forward_now - update clock to the current time
  215. *
  216. * Forward the current clock to update its state since the last call to
  217. * update_wall_time(). This is useful before significant clock changes,
  218. * as it avoids having to deal with this time offset explicitly.
  219. */
  220. static void timekeeping_forward_now(struct timekeeper *tk)
  221. {
  222. cycle_t cycle_now, cycle_delta;
  223. struct clocksource *clock;
  224. s64 nsec;
  225. clock = tk->clock;
  226. cycle_now = clock->read(clock);
  227. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  228. clock->cycle_last = cycle_now;
  229. tk->xtime_nsec += cycle_delta * tk->mult;
  230. /* If arch requires, add in get_arch_timeoffset() */
  231. tk->xtime_nsec += (u64)get_arch_timeoffset() << tk->shift;
  232. tk_normalize_xtime(tk);
  233. nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
  234. timespec_add_ns(&tk->raw_time, nsec);
  235. }
  236. /**
  237. * __getnstimeofday - Returns the time of day in a timespec.
  238. * @ts: pointer to the timespec to be set
  239. *
  240. * Updates the time of day in the timespec.
  241. * Returns 0 on success, or -ve when suspended (timespec will be undefined).
  242. */
  243. int __getnstimeofday(struct timespec *ts)
  244. {
  245. struct timekeeper *tk = &timekeeper;
  246. unsigned long seq;
  247. s64 nsecs = 0;
  248. do {
  249. seq = read_seqbegin(&tk->lock);
  250. ts->tv_sec = tk->xtime_sec;
  251. nsecs = timekeeping_get_ns(tk);
  252. } while (read_seqretry(&tk->lock, seq));
  253. ts->tv_nsec = 0;
  254. timespec_add_ns(ts, nsecs);
  255. /*
  256. * Do not bail out early, in case there were callers still using
  257. * the value, even in the face of the WARN_ON.
  258. */
  259. if (unlikely(timekeeping_suspended))
  260. return -EAGAIN;
  261. return 0;
  262. }
  263. EXPORT_SYMBOL(__getnstimeofday);
  264. /**
  265. * getnstimeofday - Returns the time of day in a timespec.
  266. * @ts: pointer to the timespec to be set
  267. *
  268. * Returns the time of day in a timespec (WARN if suspended).
  269. */
  270. void getnstimeofday(struct timespec *ts)
  271. {
  272. WARN_ON(__getnstimeofday(ts));
  273. }
  274. EXPORT_SYMBOL(getnstimeofday);
  275. ktime_t ktime_get(void)
  276. {
  277. struct timekeeper *tk = &timekeeper;
  278. unsigned int seq;
  279. s64 secs, nsecs;
  280. WARN_ON(timekeeping_suspended);
  281. do {
  282. seq = read_seqbegin(&tk->lock);
  283. secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
  284. nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec;
  285. } while (read_seqretry(&tk->lock, seq));
  286. /*
  287. * Use ktime_set/ktime_add_ns to create a proper ktime on
  288. * 32-bit architectures without CONFIG_KTIME_SCALAR.
  289. */
  290. return ktime_add_ns(ktime_set(secs, 0), nsecs);
  291. }
  292. EXPORT_SYMBOL_GPL(ktime_get);
  293. /**
  294. * ktime_get_ts - get the monotonic clock in timespec format
  295. * @ts: pointer to timespec variable
  296. *
  297. * The function calculates the monotonic clock from the realtime
  298. * clock and the wall_to_monotonic offset and stores the result
  299. * in normalized timespec format in the variable pointed to by @ts.
  300. */
  301. void ktime_get_ts(struct timespec *ts)
  302. {
  303. struct timekeeper *tk = &timekeeper;
  304. struct timespec tomono;
  305. s64 nsec;
  306. unsigned int seq;
  307. WARN_ON(timekeeping_suspended);
  308. do {
  309. seq = read_seqbegin(&tk->lock);
  310. ts->tv_sec = tk->xtime_sec;
  311. nsec = timekeeping_get_ns(tk);
  312. tomono = tk->wall_to_monotonic;
  313. } while (read_seqretry(&tk->lock, seq));
  314. ts->tv_sec += tomono.tv_sec;
  315. ts->tv_nsec = 0;
  316. timespec_add_ns(ts, nsec + tomono.tv_nsec);
  317. }
  318. EXPORT_SYMBOL_GPL(ktime_get_ts);
  319. /**
  320. * timekeeping_clocktai - Returns the TAI time of day in a timespec
  321. * @ts: pointer to the timespec to be set
  322. *
  323. * Returns the time of day in a timespec.
  324. */
  325. void timekeeping_clocktai(struct timespec *ts)
  326. {
  327. struct timekeeper *tk = &timekeeper;
  328. unsigned long seq;
  329. u64 nsecs;
  330. WARN_ON(timekeeping_suspended);
  331. do {
  332. seq = read_seqbegin(&tk->lock);
  333. ts->tv_sec = tk->xtime_sec + tk->tai_offset;
  334. nsecs = timekeeping_get_ns(tk);
  335. } while (read_seqretry(&tk->lock, seq));
  336. ts->tv_nsec = 0;
  337. timespec_add_ns(ts, nsecs);
  338. }
  339. EXPORT_SYMBOL(timekeeping_clocktai);
  340. /**
  341. * ktime_get_clocktai - Returns the TAI time of day in a ktime
  342. *
  343. * Returns the time of day in a ktime.
  344. */
  345. ktime_t ktime_get_clocktai(void)
  346. {
  347. struct timespec ts;
  348. timekeeping_clocktai(&ts);
  349. return timespec_to_ktime(ts);
  350. }
  351. EXPORT_SYMBOL(ktime_get_clocktai);
  352. #ifdef CONFIG_NTP_PPS
  353. /**
  354. * getnstime_raw_and_real - get day and raw monotonic time in timespec format
  355. * @ts_raw: pointer to the timespec to be set to raw monotonic time
  356. * @ts_real: pointer to the timespec to be set to the time of day
  357. *
  358. * This function reads both the time of day and raw monotonic time at the
  359. * same time atomically and stores the resulting timestamps in timespec
  360. * format.
  361. */
  362. void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
  363. {
  364. struct timekeeper *tk = &timekeeper;
  365. unsigned long seq;
  366. s64 nsecs_raw, nsecs_real;
  367. WARN_ON_ONCE(timekeeping_suspended);
  368. do {
  369. seq = read_seqbegin(&tk->lock);
  370. *ts_raw = tk->raw_time;
  371. ts_real->tv_sec = tk->xtime_sec;
  372. ts_real->tv_nsec = 0;
  373. nsecs_raw = timekeeping_get_ns_raw(tk);
  374. nsecs_real = timekeeping_get_ns(tk);
  375. } while (read_seqretry(&tk->lock, seq));
  376. timespec_add_ns(ts_raw, nsecs_raw);
  377. timespec_add_ns(ts_real, nsecs_real);
  378. }
  379. EXPORT_SYMBOL(getnstime_raw_and_real);
  380. #endif /* CONFIG_NTP_PPS */
  381. /**
  382. * do_gettimeofday - Returns the time of day in a timeval
  383. * @tv: pointer to the timeval to be set
  384. *
  385. * NOTE: Users should be converted to using getnstimeofday()
  386. */
  387. void do_gettimeofday(struct timeval *tv)
  388. {
  389. struct timespec now;
  390. getnstimeofday(&now);
  391. tv->tv_sec = now.tv_sec;
  392. tv->tv_usec = now.tv_nsec/1000;
  393. }
  394. EXPORT_SYMBOL(do_gettimeofday);
  395. /**
  396. * do_settimeofday - Sets the time of day
  397. * @tv: pointer to the timespec variable containing the new time
  398. *
  399. * Sets the time of day to the new time and update NTP and notify hrtimers
  400. */
  401. int do_settimeofday(const struct timespec *tv)
  402. {
  403. struct timekeeper *tk = &timekeeper;
  404. struct timespec ts_delta, xt;
  405. unsigned long flags;
  406. if (!timespec_valid_strict(tv))
  407. return -EINVAL;
  408. write_seqlock_irqsave(&tk->lock, flags);
  409. timekeeping_forward_now(tk);
  410. xt = tk_xtime(tk);
  411. ts_delta.tv_sec = tv->tv_sec - xt.tv_sec;
  412. ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec;
  413. tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta));
  414. tk_set_xtime(tk, tv);
  415. timekeeping_update(tk, true);
  416. write_sequnlock_irqrestore(&tk->lock, flags);
  417. /* signal hrtimers about time change */
  418. clock_was_set();
  419. return 0;
  420. }
  421. EXPORT_SYMBOL(do_settimeofday);
  422. /**
  423. * timekeeping_inject_offset - Adds or subtracts from the current time.
  424. * @tv: pointer to the timespec variable containing the offset
  425. *
  426. * Adds or subtracts an offset value from the current time.
  427. */
  428. int timekeeping_inject_offset(struct timespec *ts)
  429. {
  430. struct timekeeper *tk = &timekeeper;
  431. unsigned long flags;
  432. struct timespec tmp;
  433. int ret = 0;
  434. if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
  435. return -EINVAL;
  436. write_seqlock_irqsave(&tk->lock, flags);
  437. timekeeping_forward_now(tk);
  438. /* Make sure the proposed value is valid */
  439. tmp = timespec_add(tk_xtime(tk), *ts);
  440. if (!timespec_valid_strict(&tmp)) {
  441. ret = -EINVAL;
  442. goto error;
  443. }
  444. tk_xtime_add(tk, ts);
  445. tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts));
  446. error: /* even if we error out, we forwarded the time, so call update */
  447. timekeeping_update(tk, true);
  448. write_sequnlock_irqrestore(&tk->lock, flags);
  449. /* signal hrtimers about time change */
  450. clock_was_set();
  451. return ret;
  452. }
  453. EXPORT_SYMBOL(timekeeping_inject_offset);
  454. /**
  455. * timekeeping_get_tai_offset - Returns current TAI offset from UTC
  456. *
  457. */
  458. s32 timekeeping_get_tai_offset(void)
  459. {
  460. struct timekeeper *tk = &timekeeper;
  461. unsigned int seq;
  462. s32 ret;
  463. do {
  464. seq = read_seqbegin(&tk->lock);
  465. ret = tk->tai_offset;
  466. } while (read_seqretry(&tk->lock, seq));
  467. return ret;
  468. }
  469. /**
  470. * __timekeeping_set_tai_offset - Lock free worker function
  471. *
  472. */
  473. void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
  474. {
  475. tk->tai_offset = tai_offset;
  476. tk->offs_tai = ktime_sub(tk->offs_real, ktime_set(tai_offset, 0));
  477. }
  478. /**
  479. * timekeeping_set_tai_offset - Sets the current TAI offset from UTC
  480. *
  481. */
  482. void timekeeping_set_tai_offset(s32 tai_offset)
  483. {
  484. struct timekeeper *tk = &timekeeper;
  485. unsigned long flags;
  486. write_seqlock_irqsave(&tk->lock, flags);
  487. __timekeeping_set_tai_offset(tk, tai_offset);
  488. write_sequnlock_irqrestore(&tk->lock, flags);
  489. }
  490. /**
  491. * change_clocksource - Swaps clocksources if a new one is available
  492. *
  493. * Accumulates current time interval and initializes new clocksource
  494. */
  495. static int change_clocksource(void *data)
  496. {
  497. struct timekeeper *tk = &timekeeper;
  498. struct clocksource *new, *old;
  499. unsigned long flags;
  500. new = (struct clocksource *) data;
  501. write_seqlock_irqsave(&tk->lock, flags);
  502. timekeeping_forward_now(tk);
  503. if (!new->enable || new->enable(new) == 0) {
  504. old = tk->clock;
  505. tk_setup_internals(tk, new);
  506. if (old->disable)
  507. old->disable(old);
  508. }
  509. timekeeping_update(tk, true);
  510. write_sequnlock_irqrestore(&tk->lock, flags);
  511. return 0;
  512. }
  513. /**
  514. * timekeeping_notify - Install a new clock source
  515. * @clock: pointer to the clock source
  516. *
  517. * This function is called from clocksource.c after a new, better clock
  518. * source has been registered. The caller holds the clocksource_mutex.
  519. */
  520. void timekeeping_notify(struct clocksource *clock)
  521. {
  522. struct timekeeper *tk = &timekeeper;
  523. if (tk->clock == clock)
  524. return;
  525. stop_machine(change_clocksource, clock, NULL);
  526. tick_clock_notify();
  527. }
  528. /**
  529. * ktime_get_real - get the real (wall-) time in ktime_t format
  530. *
  531. * returns the time in ktime_t format
  532. */
  533. ktime_t ktime_get_real(void)
  534. {
  535. struct timespec now;
  536. getnstimeofday(&now);
  537. return timespec_to_ktime(now);
  538. }
  539. EXPORT_SYMBOL_GPL(ktime_get_real);
  540. /**
  541. * getrawmonotonic - Returns the raw monotonic time in a timespec
  542. * @ts: pointer to the timespec to be set
  543. *
  544. * Returns the raw monotonic time (completely un-modified by ntp)
  545. */
  546. void getrawmonotonic(struct timespec *ts)
  547. {
  548. struct timekeeper *tk = &timekeeper;
  549. unsigned long seq;
  550. s64 nsecs;
  551. do {
  552. seq = read_seqbegin(&tk->lock);
  553. nsecs = timekeeping_get_ns_raw(tk);
  554. *ts = tk->raw_time;
  555. } while (read_seqretry(&tk->lock, seq));
  556. timespec_add_ns(ts, nsecs);
  557. }
  558. EXPORT_SYMBOL(getrawmonotonic);
  559. /**
  560. * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
  561. */
  562. int timekeeping_valid_for_hres(void)
  563. {
  564. struct timekeeper *tk = &timekeeper;
  565. unsigned long seq;
  566. int ret;
  567. do {
  568. seq = read_seqbegin(&tk->lock);
  569. ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
  570. } while (read_seqretry(&tk->lock, seq));
  571. return ret;
  572. }
  573. /**
  574. * timekeeping_max_deferment - Returns max time the clocksource can be deferred
  575. */
  576. u64 timekeeping_max_deferment(void)
  577. {
  578. struct timekeeper *tk = &timekeeper;
  579. unsigned long seq;
  580. u64 ret;
  581. do {
  582. seq = read_seqbegin(&tk->lock);
  583. ret = tk->clock->max_idle_ns;
  584. } while (read_seqretry(&tk->lock, seq));
  585. return ret;
  586. }
  587. /**
  588. * read_persistent_clock - Return time from the persistent clock.
  589. *
  590. * Weak dummy function for arches that do not yet support it.
  591. * Reads the time from the battery backed persistent clock.
  592. * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
  593. *
  594. * XXX - Do be sure to remove it once all arches implement it.
  595. */
  596. void __attribute__((weak)) read_persistent_clock(struct timespec *ts)
  597. {
  598. ts->tv_sec = 0;
  599. ts->tv_nsec = 0;
  600. }
  601. /**
  602. * read_boot_clock - Return time of the system start.
  603. *
  604. * Weak dummy function for arches that do not yet support it.
  605. * Function to read the exact time the system has been started.
  606. * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
  607. *
  608. * XXX - Do be sure to remove it once all arches implement it.
  609. */
  610. void __attribute__((weak)) read_boot_clock(struct timespec *ts)
  611. {
  612. ts->tv_sec = 0;
  613. ts->tv_nsec = 0;
  614. }
  615. /*
  616. * timekeeping_init - Initializes the clocksource and common timekeeping values
  617. */
  618. void __init timekeeping_init(void)
  619. {
  620. struct timekeeper *tk = &timekeeper;
  621. struct clocksource *clock;
  622. unsigned long flags;
  623. struct timespec now, boot, tmp;
  624. read_persistent_clock(&now);
  625. if (!timespec_valid_strict(&now)) {
  626. pr_warn("WARNING: Persistent clock returned invalid value!\n"
  627. " Check your CMOS/BIOS settings.\n");
  628. now.tv_sec = 0;
  629. now.tv_nsec = 0;
  630. } else if (now.tv_sec || now.tv_nsec)
  631. persistent_clock_exist = true;
  632. read_boot_clock(&boot);
  633. if (!timespec_valid_strict(&boot)) {
  634. pr_warn("WARNING: Boot clock returned invalid value!\n"
  635. " Check your CMOS/BIOS settings.\n");
  636. boot.tv_sec = 0;
  637. boot.tv_nsec = 0;
  638. }
  639. seqlock_init(&tk->lock);
  640. ntp_init();
  641. write_seqlock_irqsave(&tk->lock, flags);
  642. clock = clocksource_default_clock();
  643. if (clock->enable)
  644. clock->enable(clock);
  645. tk_setup_internals(tk, clock);
  646. tk_set_xtime(tk, &now);
  647. tk->raw_time.tv_sec = 0;
  648. tk->raw_time.tv_nsec = 0;
  649. if (boot.tv_sec == 0 && boot.tv_nsec == 0)
  650. boot = tk_xtime(tk);
  651. set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec);
  652. tk_set_wall_to_mono(tk, tmp);
  653. tmp.tv_sec = 0;
  654. tmp.tv_nsec = 0;
  655. tk_set_sleep_time(tk, tmp);
  656. write_sequnlock_irqrestore(&tk->lock, flags);
  657. }
  658. /* time in seconds when suspend began */
  659. static struct timespec timekeeping_suspend_time;
  660. /**
  661. * __timekeeping_inject_sleeptime - Internal function to add sleep interval
  662. * @delta: pointer to a timespec delta value
  663. *
  664. * Takes a timespec offset measuring a suspend interval and properly
  665. * adds the sleep offset to the timekeeping variables.
  666. */
  667. static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
  668. struct timespec *delta)
  669. {
  670. if (!timespec_valid_strict(delta)) {
  671. printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid "
  672. "sleep delta value!\n");
  673. return;
  674. }
  675. tk_xtime_add(tk, delta);
  676. tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *delta));
  677. tk_set_sleep_time(tk, timespec_add(tk->total_sleep_time, *delta));
  678. }
  679. /**
  680. * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values
  681. * @delta: pointer to a timespec delta value
  682. *
  683. * This hook is for architectures that cannot support read_persistent_clock
  684. * because their RTC/persistent clock is only accessible when irqs are enabled.
  685. *
  686. * This function should only be called by rtc_resume(), and allows
  687. * a suspend offset to be injected into the timekeeping values.
  688. */
  689. void timekeeping_inject_sleeptime(struct timespec *delta)
  690. {
  691. struct timekeeper *tk = &timekeeper;
  692. unsigned long flags;
  693. /*
  694. * Make sure we don't set the clock twice, as timekeeping_resume()
  695. * already did it
  696. */
  697. if (has_persistent_clock())
  698. return;
  699. write_seqlock_irqsave(&tk->lock, flags);
  700. timekeeping_forward_now(tk);
  701. __timekeeping_inject_sleeptime(tk, delta);
  702. timekeeping_update(tk, true);
  703. write_sequnlock_irqrestore(&tk->lock, flags);
  704. /* signal hrtimers about time change */
  705. clock_was_set();
  706. }
  707. /**
  708. * timekeeping_resume - Resumes the generic timekeeping subsystem.
  709. *
  710. * This is for the generic clocksource timekeeping.
  711. * xtime/wall_to_monotonic/jiffies/etc are
  712. * still managed by arch specific suspend/resume code.
  713. */
  714. static void timekeeping_resume(void)
  715. {
  716. struct timekeeper *tk = &timekeeper;
  717. struct clocksource *clock = tk->clock;
  718. unsigned long flags;
  719. struct timespec ts_new, ts_delta;
  720. cycle_t cycle_now, cycle_delta;
  721. bool suspendtime_found = false;
  722. read_persistent_clock(&ts_new);
  723. clockevents_resume();
  724. clocksource_resume();
  725. write_seqlock_irqsave(&tk->lock, flags);
  726. /*
  727. * After system resumes, we need to calculate the suspended time and
  728. * compensate it for the OS time. There are 3 sources that could be
  729. * used: Nonstop clocksource during suspend, persistent clock and rtc
  730. * device.
  731. *
  732. * One specific platform may have 1 or 2 or all of them, and the
  733. * preference will be:
  734. * suspend-nonstop clocksource -> persistent clock -> rtc
  735. * The less preferred source will only be tried if there is no better
  736. * usable source. The rtc part is handled separately in rtc core code.
  737. */
  738. cycle_now = clock->read(clock);
  739. if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) &&
  740. cycle_now > clock->cycle_last) {
  741. u64 num, max = ULLONG_MAX;
  742. u32 mult = clock->mult;
  743. u32 shift = clock->shift;
  744. s64 nsec = 0;
  745. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  746. /*
  747. * "cycle_delta * mutl" may cause 64 bits overflow, if the
  748. * suspended time is too long. In that case we need do the
  749. * 64 bits math carefully
  750. */
  751. do_div(max, mult);
  752. if (cycle_delta > max) {
  753. num = div64_u64(cycle_delta, max);
  754. nsec = (((u64) max * mult) >> shift) * num;
  755. cycle_delta -= num * max;
  756. }
  757. nsec += ((u64) cycle_delta * mult) >> shift;
  758. ts_delta = ns_to_timespec(nsec);
  759. suspendtime_found = true;
  760. } else if (timespec_compare(&ts_new, &timekeeping_suspend_time) > 0) {
  761. ts_delta = timespec_sub(ts_new, timekeeping_suspend_time);
  762. suspendtime_found = true;
  763. }
  764. if (suspendtime_found)
  765. __timekeeping_inject_sleeptime(tk, &ts_delta);
  766. /* Re-base the last cycle value */
  767. clock->cycle_last = cycle_now;
  768. tk->ntp_error = 0;
  769. timekeeping_suspended = 0;
  770. timekeeping_update(tk, false);
  771. write_sequnlock_irqrestore(&tk->lock, flags);
  772. touch_softlockup_watchdog();
  773. clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
  774. /* Resume hrtimers */
  775. hrtimers_resume();
  776. }
  777. static int timekeeping_suspend(void)
  778. {
  779. struct timekeeper *tk = &timekeeper;
  780. unsigned long flags;
  781. struct timespec delta, delta_delta;
  782. static struct timespec old_delta;
  783. read_persistent_clock(&timekeeping_suspend_time);
  784. write_seqlock_irqsave(&tk->lock, flags);
  785. timekeeping_forward_now(tk);
  786. timekeeping_suspended = 1;
  787. /*
  788. * To avoid drift caused by repeated suspend/resumes,
  789. * which each can add ~1 second drift error,
  790. * try to compensate so the difference in system time
  791. * and persistent_clock time stays close to constant.
  792. */
  793. delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time);
  794. delta_delta = timespec_sub(delta, old_delta);
  795. if (abs(delta_delta.tv_sec) >= 2) {
  796. /*
  797. * if delta_delta is too large, assume time correction
  798. * has occured and set old_delta to the current delta.
  799. */
  800. old_delta = delta;
  801. } else {
  802. /* Otherwise try to adjust old_system to compensate */
  803. timekeeping_suspend_time =
  804. timespec_add(timekeeping_suspend_time, delta_delta);
  805. }
  806. write_sequnlock_irqrestore(&tk->lock, flags);
  807. clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
  808. clocksource_suspend();
  809. clockevents_suspend();
  810. return 0;
  811. }
  812. /* sysfs resume/suspend bits for timekeeping */
  813. static struct syscore_ops timekeeping_syscore_ops = {
  814. .resume = timekeeping_resume,
  815. .suspend = timekeeping_suspend,
  816. };
  817. static int __init timekeeping_init_ops(void)
  818. {
  819. register_syscore_ops(&timekeeping_syscore_ops);
  820. return 0;
  821. }
  822. device_initcall(timekeeping_init_ops);
  823. /*
  824. * If the error is already larger, we look ahead even further
  825. * to compensate for late or lost adjustments.
  826. */
  827. static __always_inline int timekeeping_bigadjust(struct timekeeper *tk,
  828. s64 error, s64 *interval,
  829. s64 *offset)
  830. {
  831. s64 tick_error, i;
  832. u32 look_ahead, adj;
  833. s32 error2, mult;
  834. /*
  835. * Use the current error value to determine how much to look ahead.
  836. * The larger the error the slower we adjust for it to avoid problems
  837. * with losing too many ticks, otherwise we would overadjust and
  838. * produce an even larger error. The smaller the adjustment the
  839. * faster we try to adjust for it, as lost ticks can do less harm
  840. * here. This is tuned so that an error of about 1 msec is adjusted
  841. * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
  842. */
  843. error2 = tk->ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
  844. error2 = abs(error2);
  845. for (look_ahead = 0; error2 > 0; look_ahead++)
  846. error2 >>= 2;
  847. /*
  848. * Now calculate the error in (1 << look_ahead) ticks, but first
  849. * remove the single look ahead already included in the error.
  850. */
  851. tick_error = ntp_tick_length() >> (tk->ntp_error_shift + 1);
  852. tick_error -= tk->xtime_interval >> 1;
  853. error = ((error - tick_error) >> look_ahead) + tick_error;
  854. /* Finally calculate the adjustment shift value. */
  855. i = *interval;
  856. mult = 1;
  857. if (error < 0) {
  858. error = -error;
  859. *interval = -*interval;
  860. *offset = -*offset;
  861. mult = -1;
  862. }
  863. for (adj = 0; error > i; adj++)
  864. error >>= 1;
  865. *interval <<= adj;
  866. *offset <<= adj;
  867. return mult << adj;
  868. }
  869. /*
  870. * Adjust the multiplier to reduce the error value,
  871. * this is optimized for the most common adjustments of -1,0,1,
  872. * for other values we can do a bit more work.
  873. */
  874. static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
  875. {
  876. s64 error, interval = tk->cycle_interval;
  877. int adj;
  878. /*
  879. * The point of this is to check if the error is greater than half
  880. * an interval.
  881. *
  882. * First we shift it down from NTP_SHIFT to clocksource->shifted nsecs.
  883. *
  884. * Note we subtract one in the shift, so that error is really error*2.
  885. * This "saves" dividing(shifting) interval twice, but keeps the
  886. * (error > interval) comparison as still measuring if error is
  887. * larger than half an interval.
  888. *
  889. * Note: It does not "save" on aggravation when reading the code.
  890. */
  891. error = tk->ntp_error >> (tk->ntp_error_shift - 1);
  892. if (error > interval) {
  893. /*
  894. * We now divide error by 4(via shift), which checks if
  895. * the error is greater than twice the interval.
  896. * If it is greater, we need a bigadjust, if its smaller,
  897. * we can adjust by 1.
  898. */
  899. error >>= 2;
  900. /*
  901. * XXX - In update_wall_time, we round up to the next
  902. * nanosecond, and store the amount rounded up into
  903. * the error. This causes the likely below to be unlikely.
  904. *
  905. * The proper fix is to avoid rounding up by using
  906. * the high precision tk->xtime_nsec instead of
  907. * xtime.tv_nsec everywhere. Fixing this will take some
  908. * time.
  909. */
  910. if (likely(error <= interval))
  911. adj = 1;
  912. else
  913. adj = timekeeping_bigadjust(tk, error, &interval, &offset);
  914. } else {
  915. if (error < -interval) {
  916. /* See comment above, this is just switched for the negative */
  917. error >>= 2;
  918. if (likely(error >= -interval)) {
  919. adj = -1;
  920. interval = -interval;
  921. offset = -offset;
  922. } else {
  923. adj = timekeeping_bigadjust(tk, error, &interval, &offset);
  924. }
  925. } else {
  926. goto out_adjust;
  927. }
  928. }
  929. if (unlikely(tk->clock->maxadj &&
  930. (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) {
  931. printk_once(KERN_WARNING
  932. "Adjusting %s more than 11%% (%ld vs %ld)\n",
  933. tk->clock->name, (long)tk->mult + adj,
  934. (long)tk->clock->mult + tk->clock->maxadj);
  935. }
  936. /*
  937. * So the following can be confusing.
  938. *
  939. * To keep things simple, lets assume adj == 1 for now.
  940. *
  941. * When adj != 1, remember that the interval and offset values
  942. * have been appropriately scaled so the math is the same.
  943. *
  944. * The basic idea here is that we're increasing the multiplier
  945. * by one, this causes the xtime_interval to be incremented by
  946. * one cycle_interval. This is because:
  947. * xtime_interval = cycle_interval * mult
  948. * So if mult is being incremented by one:
  949. * xtime_interval = cycle_interval * (mult + 1)
  950. * Its the same as:
  951. * xtime_interval = (cycle_interval * mult) + cycle_interval
  952. * Which can be shortened to:
  953. * xtime_interval += cycle_interval
  954. *
  955. * So offset stores the non-accumulated cycles. Thus the current
  956. * time (in shifted nanoseconds) is:
  957. * now = (offset * adj) + xtime_nsec
  958. * Now, even though we're adjusting the clock frequency, we have
  959. * to keep time consistent. In other words, we can't jump back
  960. * in time, and we also want to avoid jumping forward in time.
  961. *
  962. * So given the same offset value, we need the time to be the same
  963. * both before and after the freq adjustment.
  964. * now = (offset * adj_1) + xtime_nsec_1
  965. * now = (offset * adj_2) + xtime_nsec_2
  966. * So:
  967. * (offset * adj_1) + xtime_nsec_1 =
  968. * (offset * adj_2) + xtime_nsec_2
  969. * And we know:
  970. * adj_2 = adj_1 + 1
  971. * So:
  972. * (offset * adj_1) + xtime_nsec_1 =
  973. * (offset * (adj_1+1)) + xtime_nsec_2
  974. * (offset * adj_1) + xtime_nsec_1 =
  975. * (offset * adj_1) + offset + xtime_nsec_2
  976. * Canceling the sides:
  977. * xtime_nsec_1 = offset + xtime_nsec_2
  978. * Which gives us:
  979. * xtime_nsec_2 = xtime_nsec_1 - offset
  980. * Which simplfies to:
  981. * xtime_nsec -= offset
  982. *
  983. * XXX - TODO: Doc ntp_error calculation.
  984. */
  985. tk->mult += adj;
  986. tk->xtime_interval += interval;
  987. tk->xtime_nsec -= offset;
  988. tk->ntp_error -= (interval - offset) << tk->ntp_error_shift;
  989. out_adjust:
  990. /*
  991. * It may be possible that when we entered this function, xtime_nsec
  992. * was very small. Further, if we're slightly speeding the clocksource
  993. * in the code above, its possible the required corrective factor to
  994. * xtime_nsec could cause it to underflow.
  995. *
  996. * Now, since we already accumulated the second, cannot simply roll
  997. * the accumulated second back, since the NTP subsystem has been
  998. * notified via second_overflow. So instead we push xtime_nsec forward
  999. * by the amount we underflowed, and add that amount into the error.
  1000. *
  1001. * We'll correct this error next time through this function, when
  1002. * xtime_nsec is not as small.
  1003. */
  1004. if (unlikely((s64)tk->xtime_nsec < 0)) {
  1005. s64 neg = -(s64)tk->xtime_nsec;
  1006. tk->xtime_nsec = 0;
  1007. tk->ntp_error += neg << tk->ntp_error_shift;
  1008. }
  1009. }
  1010. /**
  1011. * accumulate_nsecs_to_secs - Accumulates nsecs into secs
  1012. *
  1013. * Helper function that accumulates a the nsecs greater then a second
  1014. * from the xtime_nsec field to the xtime_secs field.
  1015. * It also calls into the NTP code to handle leapsecond processing.
  1016. *
  1017. */
  1018. static inline void accumulate_nsecs_to_secs(struct timekeeper *tk)
  1019. {
  1020. u64 nsecps = (u64)NSEC_PER_SEC << tk->shift;
  1021. while (tk->xtime_nsec >= nsecps) {
  1022. int leap;
  1023. tk->xtime_nsec -= nsecps;
  1024. tk->xtime_sec++;
  1025. /* Figure out if its a leap sec and apply if needed */
  1026. leap = second_overflow(tk->xtime_sec);
  1027. if (unlikely(leap)) {
  1028. struct timespec ts;
  1029. tk->xtime_sec += leap;
  1030. ts.tv_sec = leap;
  1031. ts.tv_nsec = 0;
  1032. tk_set_wall_to_mono(tk,
  1033. timespec_sub(tk->wall_to_monotonic, ts));
  1034. __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
  1035. clock_was_set_delayed();
  1036. }
  1037. }
  1038. }
  1039. /**
  1040. * logarithmic_accumulation - shifted accumulation of cycles
  1041. *
  1042. * This functions accumulates a shifted interval of cycles into
  1043. * into a shifted interval nanoseconds. Allows for O(log) accumulation
  1044. * loop.
  1045. *
  1046. * Returns the unconsumed cycles.
  1047. */
  1048. static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
  1049. u32 shift)
  1050. {
  1051. cycle_t interval = tk->cycle_interval << shift;
  1052. u64 raw_nsecs;
  1053. /* If the offset is smaller then a shifted interval, do nothing */
  1054. if (offset < interval)
  1055. return offset;
  1056. /* Accumulate one shifted interval */
  1057. offset -= interval;
  1058. tk->clock->cycle_last += interval;
  1059. tk->xtime_nsec += tk->xtime_interval << shift;
  1060. accumulate_nsecs_to_secs(tk);
  1061. /* Accumulate raw time */
  1062. raw_nsecs = (u64)tk->raw_interval << shift;
  1063. raw_nsecs += tk->raw_time.tv_nsec;
  1064. if (raw_nsecs >= NSEC_PER_SEC) {
  1065. u64 raw_secs = raw_nsecs;
  1066. raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
  1067. tk->raw_time.tv_sec += raw_secs;
  1068. }
  1069. tk->raw_time.tv_nsec = raw_nsecs;
  1070. /* Accumulate error between NTP and clock interval */
  1071. tk->ntp_error += ntp_tick_length() << shift;
  1072. tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
  1073. (tk->ntp_error_shift + shift);
  1074. return offset;
  1075. }
  1076. #ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
  1077. static inline void old_vsyscall_fixup(struct timekeeper *tk)
  1078. {
  1079. s64 remainder;
  1080. /*
  1081. * Store only full nanoseconds into xtime_nsec after rounding
  1082. * it up and add the remainder to the error difference.
  1083. * XXX - This is necessary to avoid small 1ns inconsistnecies caused
  1084. * by truncating the remainder in vsyscalls. However, it causes
  1085. * additional work to be done in timekeeping_adjust(). Once
  1086. * the vsyscall implementations are converted to use xtime_nsec
  1087. * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD
  1088. * users are removed, this can be killed.
  1089. */
  1090. remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1);
  1091. tk->xtime_nsec -= remainder;
  1092. tk->xtime_nsec += 1ULL << tk->shift;
  1093. tk->ntp_error += remainder << tk->ntp_error_shift;
  1094. }
  1095. #else
  1096. #define old_vsyscall_fixup(tk)
  1097. #endif
  1098. /**
  1099. * update_wall_time - Uses the current clocksource to increment the wall time
  1100. *
  1101. */
  1102. static void update_wall_time(void)
  1103. {
  1104. struct clocksource *clock;
  1105. struct timekeeper *tk = &timekeeper;
  1106. cycle_t offset;
  1107. int shift = 0, maxshift;
  1108. unsigned long flags;
  1109. write_seqlock_irqsave(&tk->lock, flags);
  1110. /* Make sure we're fully resumed: */
  1111. if (unlikely(timekeeping_suspended))
  1112. goto out;
  1113. clock = tk->clock;
  1114. #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
  1115. offset = tk->cycle_interval;
  1116. #else
  1117. offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
  1118. #endif
  1119. /* Check if there's really nothing to do */
  1120. if (offset < tk->cycle_interval)
  1121. goto out;
  1122. /*
  1123. * With NO_HZ we may have to accumulate many cycle_intervals
  1124. * (think "ticks") worth of time at once. To do this efficiently,
  1125. * we calculate the largest doubling multiple of cycle_intervals
  1126. * that is smaller than the offset. We then accumulate that
  1127. * chunk in one go, and then try to consume the next smaller
  1128. * doubled multiple.
  1129. */
  1130. shift = ilog2(offset) - ilog2(tk->cycle_interval);
  1131. shift = max(0, shift);
  1132. /* Bound shift to one less than what overflows tick_length */
  1133. maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
  1134. shift = min(shift, maxshift);
  1135. while (offset >= tk->cycle_interval) {
  1136. offset = logarithmic_accumulation(tk, offset, shift);
  1137. if (offset < tk->cycle_interval<<shift)
  1138. shift--;
  1139. }
  1140. /* correct the clock when NTP error is too big */
  1141. timekeeping_adjust(tk, offset);
  1142. /*
  1143. * XXX This can be killed once everyone converts
  1144. * to the new update_vsyscall.
  1145. */
  1146. old_vsyscall_fixup(tk);
  1147. /*
  1148. * Finally, make sure that after the rounding
  1149. * xtime_nsec isn't larger than NSEC_PER_SEC
  1150. */
  1151. accumulate_nsecs_to_secs(tk);
  1152. timekeeping_update(tk, false);
  1153. out:
  1154. write_sequnlock_irqrestore(&tk->lock, flags);
  1155. }
  1156. /**
  1157. * getboottime - Return the real time of system boot.
  1158. * @ts: pointer to the timespec to be set
  1159. *
  1160. * Returns the wall-time of boot in a timespec.
  1161. *
  1162. * This is based on the wall_to_monotonic offset and the total suspend
  1163. * time. Calls to settimeofday will affect the value returned (which
  1164. * basically means that however wrong your real time clock is at boot time,
  1165. * you get the right time here).
  1166. */
  1167. void getboottime(struct timespec *ts)
  1168. {
  1169. struct timekeeper *tk = &timekeeper;
  1170. struct timespec boottime = {
  1171. .tv_sec = tk->wall_to_monotonic.tv_sec +
  1172. tk->total_sleep_time.tv_sec,
  1173. .tv_nsec = tk->wall_to_monotonic.tv_nsec +
  1174. tk->total_sleep_time.tv_nsec
  1175. };
  1176. set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec);
  1177. }
  1178. EXPORT_SYMBOL_GPL(getboottime);
  1179. /**
  1180. * get_monotonic_boottime - Returns monotonic time since boot
  1181. * @ts: pointer to the timespec to be set
  1182. *
  1183. * Returns the monotonic time since boot in a timespec.
  1184. *
  1185. * This is similar to CLOCK_MONTONIC/ktime_get_ts, but also
  1186. * includes the time spent in suspend.
  1187. */
  1188. void get_monotonic_boottime(struct timespec *ts)
  1189. {
  1190. struct timekeeper *tk = &timekeeper;
  1191. struct timespec tomono, sleep;
  1192. s64 nsec;
  1193. unsigned int seq;
  1194. WARN_ON(timekeeping_suspended);
  1195. do {
  1196. seq = read_seqbegin(&tk->lock);
  1197. ts->tv_sec = tk->xtime_sec;
  1198. nsec = timekeeping_get_ns(tk);
  1199. tomono = tk->wall_to_monotonic;
  1200. sleep = tk->total_sleep_time;
  1201. } while (read_seqretry(&tk->lock, seq));
  1202. ts->tv_sec += tomono.tv_sec + sleep.tv_sec;
  1203. ts->tv_nsec = 0;
  1204. timespec_add_ns(ts, nsec + tomono.tv_nsec + sleep.tv_nsec);
  1205. }
  1206. EXPORT_SYMBOL_GPL(get_monotonic_boottime);
  1207. /**
  1208. * ktime_get_boottime - Returns monotonic time since boot in a ktime
  1209. *
  1210. * Returns the monotonic time since boot in a ktime
  1211. *
  1212. * This is similar to CLOCK_MONTONIC/ktime_get, but also
  1213. * includes the time spent in suspend.
  1214. */
  1215. ktime_t ktime_get_boottime(void)
  1216. {
  1217. struct timespec ts;
  1218. get_monotonic_boottime(&ts);
  1219. return timespec_to_ktime(ts);
  1220. }
  1221. EXPORT_SYMBOL_GPL(ktime_get_boottime);
  1222. /**
  1223. * monotonic_to_bootbased - Convert the monotonic time to boot based.
  1224. * @ts: pointer to the timespec to be converted
  1225. */
  1226. void monotonic_to_bootbased(struct timespec *ts)
  1227. {
  1228. struct timekeeper *tk = &timekeeper;
  1229. *ts = timespec_add(*ts, tk->total_sleep_time);
  1230. }
  1231. EXPORT_SYMBOL_GPL(monotonic_to_bootbased);
  1232. unsigned long get_seconds(void)
  1233. {
  1234. struct timekeeper *tk = &timekeeper;
  1235. return tk->xtime_sec;
  1236. }
  1237. EXPORT_SYMBOL(get_seconds);
  1238. struct timespec __current_kernel_time(void)
  1239. {
  1240. struct timekeeper *tk = &timekeeper;
  1241. return tk_xtime(tk);
  1242. }
  1243. struct timespec current_kernel_time(void)
  1244. {
  1245. struct timekeeper *tk = &timekeeper;
  1246. struct timespec now;
  1247. unsigned long seq;
  1248. do {
  1249. seq = read_seqbegin(&tk->lock);
  1250. now = tk_xtime(tk);
  1251. } while (read_seqretry(&tk->lock, seq));
  1252. return now;
  1253. }
  1254. EXPORT_SYMBOL(current_kernel_time);
  1255. struct timespec get_monotonic_coarse(void)
  1256. {
  1257. struct timekeeper *tk = &timekeeper;
  1258. struct timespec now, mono;
  1259. unsigned long seq;
  1260. do {
  1261. seq = read_seqbegin(&tk->lock);
  1262. now = tk_xtime(tk);
  1263. mono = tk->wall_to_monotonic;
  1264. } while (read_seqretry(&tk->lock, seq));
  1265. set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
  1266. now.tv_nsec + mono.tv_nsec);
  1267. return now;
  1268. }
  1269. /*
  1270. * Must hold jiffies_lock
  1271. */
  1272. void do_timer(unsigned long ticks)
  1273. {
  1274. jiffies_64 += ticks;
  1275. update_wall_time();
  1276. calc_global_load(ticks);
  1277. }
  1278. /**
  1279. * get_xtime_and_monotonic_and_sleep_offset() - get xtime, wall_to_monotonic,
  1280. * and sleep offsets.
  1281. * @xtim: pointer to timespec to be set with xtime
  1282. * @wtom: pointer to timespec to be set with wall_to_monotonic
  1283. * @sleep: pointer to timespec to be set with time in suspend
  1284. */
  1285. void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
  1286. struct timespec *wtom, struct timespec *sleep)
  1287. {
  1288. struct timekeeper *tk = &timekeeper;
  1289. unsigned long seq;
  1290. do {
  1291. seq = read_seqbegin(&tk->lock);
  1292. *xtim = tk_xtime(tk);
  1293. *wtom = tk->wall_to_monotonic;
  1294. *sleep = tk->total_sleep_time;
  1295. } while (read_seqretry(&tk->lock, seq));
  1296. }
  1297. #ifdef CONFIG_HIGH_RES_TIMERS
  1298. /**
  1299. * ktime_get_update_offsets - hrtimer helper
  1300. * @offs_real: pointer to storage for monotonic -> realtime offset
  1301. * @offs_boot: pointer to storage for monotonic -> boottime offset
  1302. *
  1303. * Returns current monotonic time and updates the offsets
  1304. * Called from hrtimer_interupt() or retrigger_next_event()
  1305. */
  1306. ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot,
  1307. ktime_t *offs_tai)
  1308. {
  1309. struct timekeeper *tk = &timekeeper;
  1310. ktime_t now;
  1311. unsigned int seq;
  1312. u64 secs, nsecs;
  1313. do {
  1314. seq = read_seqbegin(&tk->lock);
  1315. secs = tk->xtime_sec;
  1316. nsecs = timekeeping_get_ns(tk);
  1317. *offs_real = tk->offs_real;
  1318. *offs_boot = tk->offs_boot;
  1319. *offs_tai = tk->offs_tai;
  1320. } while (read_seqretry(&tk->lock, seq));
  1321. now = ktime_add_ns(ktime_set(secs, 0), nsecs);
  1322. now = ktime_sub(now, *offs_real);
  1323. return now;
  1324. }
  1325. #endif
  1326. /**
  1327. * ktime_get_monotonic_offset() - get wall_to_monotonic in ktime_t format
  1328. */
  1329. ktime_t ktime_get_monotonic_offset(void)
  1330. {
  1331. struct timekeeper *tk = &timekeeper;
  1332. unsigned long seq;
  1333. struct timespec wtom;
  1334. do {
  1335. seq = read_seqbegin(&tk->lock);
  1336. wtom = tk->wall_to_monotonic;
  1337. } while (read_seqretry(&tk->lock, seq));
  1338. return timespec_to_ktime(wtom);
  1339. }
  1340. EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset);
  1341. /**
  1342. * xtime_update() - advances the timekeeping infrastructure
  1343. * @ticks: number of ticks, that have elapsed since the last call.
  1344. *
  1345. * Must be called with interrupts disabled.
  1346. */
  1347. void xtime_update(unsigned long ticks)
  1348. {
  1349. write_seqlock(&jiffies_lock);
  1350. do_timer(ticks);
  1351. write_sequnlock(&jiffies_lock);
  1352. }