hrtimer.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871
  1. /*
  2. * linux/kernel/hrtimer.c
  3. *
  4. * Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright(C) 2005, Red Hat, Inc., Ingo Molnar
  6. *
  7. * High-resolution kernel timers
  8. *
  9. * In contrast to the low-resolution timeout API implemented in
  10. * kernel/timer.c, hrtimers provide finer resolution and accuracy
  11. * depending on system configuration and capabilities.
  12. *
  13. * These timers are currently used for:
  14. * - itimers
  15. * - POSIX timers
  16. * - nanosleep
  17. * - precise in-kernel timing
  18. *
  19. * Started by: Thomas Gleixner and Ingo Molnar
  20. *
  21. * Credits:
  22. * based on kernel/timer.c
  23. *
  24. * Help, testing, suggestions, bugfixes, improvements were
  25. * provided by:
  26. *
  27. * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
  28. * et. al.
  29. *
  30. * For licencing details see kernel-base/COPYING
  31. */
  32. #include <linux/cpu.h>
  33. #include <linux/module.h>
  34. #include <linux/percpu.h>
  35. #include <linux/hrtimer.h>
  36. #include <linux/notifier.h>
  37. #include <linux/syscalls.h>
  38. #include <linux/interrupt.h>
  39. #include <asm/uaccess.h>
  40. /**
  41. * ktime_get - get the monotonic time in ktime_t format
  42. *
  43. * returns the time in ktime_t format
  44. */
  45. static ktime_t ktime_get(void)
  46. {
  47. struct timespec now;
  48. ktime_get_ts(&now);
  49. return timespec_to_ktime(now);
  50. }
  51. /**
  52. * ktime_get_real - get the real (wall-) time in ktime_t format
  53. *
  54. * returns the time in ktime_t format
  55. */
  56. static ktime_t ktime_get_real(void)
  57. {
  58. struct timespec now;
  59. getnstimeofday(&now);
  60. return timespec_to_ktime(now);
  61. }
  62. EXPORT_SYMBOL_GPL(ktime_get_real);
  63. /*
  64. * The timer bases:
  65. *
  66. * Note: If we want to add new timer bases, we have to skip the two
  67. * clock ids captured by the cpu-timers. We do this by holding empty
  68. * entries rather than doing math adjustment of the clock ids.
  69. * This ensures that we capture erroneous accesses to these clock ids
  70. * rather than moving them into the range of valid clock id's.
  71. */
  72. #define MAX_HRTIMER_BASES 2
  73. static DEFINE_PER_CPU(struct hrtimer_base, hrtimer_bases[MAX_HRTIMER_BASES]) =
  74. {
  75. {
  76. .index = CLOCK_REALTIME,
  77. .get_time = &ktime_get_real,
  78. .resolution = KTIME_REALTIME_RES,
  79. },
  80. {
  81. .index = CLOCK_MONOTONIC,
  82. .get_time = &ktime_get,
  83. .resolution = KTIME_MONOTONIC_RES,
  84. },
  85. };
  86. /**
  87. * ktime_get_ts - get the monotonic clock in timespec format
  88. *
  89. * @ts: pointer to timespec variable
  90. *
  91. * The function calculates the monotonic clock from the realtime
  92. * clock and the wall_to_monotonic offset and stores the result
  93. * in normalized timespec format in the variable pointed to by ts.
  94. */
  95. void ktime_get_ts(struct timespec *ts)
  96. {
  97. struct timespec tomono;
  98. unsigned long seq;
  99. do {
  100. seq = read_seqbegin(&xtime_lock);
  101. getnstimeofday(ts);
  102. tomono = wall_to_monotonic;
  103. } while (read_seqretry(&xtime_lock, seq));
  104. set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
  105. ts->tv_nsec + tomono.tv_nsec);
  106. }
  107. EXPORT_SYMBOL_GPL(ktime_get_ts);
  108. /*
  109. * Get the coarse grained time at the softirq based on xtime and
  110. * wall_to_monotonic.
  111. */
  112. static void hrtimer_get_softirq_time(struct hrtimer_base *base)
  113. {
  114. ktime_t xtim, tomono;
  115. unsigned long seq;
  116. do {
  117. seq = read_seqbegin(&xtime_lock);
  118. xtim = timespec_to_ktime(xtime);
  119. tomono = timespec_to_ktime(wall_to_monotonic);
  120. } while (read_seqretry(&xtime_lock, seq));
  121. base[CLOCK_REALTIME].softirq_time = xtim;
  122. base[CLOCK_MONOTONIC].softirq_time = ktime_add(xtim, tomono);
  123. }
  124. /*
  125. * Functions and macros which are different for UP/SMP systems are kept in a
  126. * single place
  127. */
  128. #ifdef CONFIG_SMP
  129. #define set_curr_timer(b, t) do { (b)->curr_timer = (t); } while (0)
  130. /*
  131. * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
  132. * means that all timers which are tied to this base via timer->base are
  133. * locked, and the base itself is locked too.
  134. *
  135. * So __run_timers/migrate_timers can safely modify all timers which could
  136. * be found on the lists/queues.
  137. *
  138. * When the timer's base is locked, and the timer removed from list, it is
  139. * possible to set timer->base = NULL and drop the lock: the timer remains
  140. * locked.
  141. */
  142. static struct hrtimer_base *lock_hrtimer_base(const struct hrtimer *timer,
  143. unsigned long *flags)
  144. {
  145. struct hrtimer_base *base;
  146. for (;;) {
  147. base = timer->base;
  148. if (likely(base != NULL)) {
  149. spin_lock_irqsave(&base->lock, *flags);
  150. if (likely(base == timer->base))
  151. return base;
  152. /* The timer has migrated to another CPU: */
  153. spin_unlock_irqrestore(&base->lock, *flags);
  154. }
  155. cpu_relax();
  156. }
  157. }
  158. /*
  159. * Switch the timer base to the current CPU when possible.
  160. */
  161. static inline struct hrtimer_base *
  162. switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_base *base)
  163. {
  164. struct hrtimer_base *new_base;
  165. new_base = &__get_cpu_var(hrtimer_bases[base->index]);
  166. if (base != new_base) {
  167. /*
  168. * We are trying to schedule the timer on the local CPU.
  169. * However we can't change timer's base while it is running,
  170. * so we keep it on the same CPU. No hassle vs. reprogramming
  171. * the event source in the high resolution case. The softirq
  172. * code will take care of this when the timer function has
  173. * completed. There is no conflict as we hold the lock until
  174. * the timer is enqueued.
  175. */
  176. if (unlikely(base->curr_timer == timer))
  177. return base;
  178. /* See the comment in lock_timer_base() */
  179. timer->base = NULL;
  180. spin_unlock(&base->lock);
  181. spin_lock(&new_base->lock);
  182. timer->base = new_base;
  183. }
  184. return new_base;
  185. }
  186. #else /* CONFIG_SMP */
  187. #define set_curr_timer(b, t) do { } while (0)
  188. static inline struct hrtimer_base *
  189. lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
  190. {
  191. struct hrtimer_base *base = timer->base;
  192. spin_lock_irqsave(&base->lock, *flags);
  193. return base;
  194. }
  195. #define switch_hrtimer_base(t, b) (b)
  196. #endif /* !CONFIG_SMP */
  197. /*
  198. * Functions for the union type storage format of ktime_t which are
  199. * too large for inlining:
  200. */
  201. #if BITS_PER_LONG < 64
  202. # ifndef CONFIG_KTIME_SCALAR
  203. /**
  204. * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable
  205. *
  206. * @kt: addend
  207. * @nsec: the scalar nsec value to add
  208. *
  209. * Returns the sum of kt and nsec in ktime_t format
  210. */
  211. ktime_t ktime_add_ns(const ktime_t kt, u64 nsec)
  212. {
  213. ktime_t tmp;
  214. if (likely(nsec < NSEC_PER_SEC)) {
  215. tmp.tv64 = nsec;
  216. } else {
  217. unsigned long rem = do_div(nsec, NSEC_PER_SEC);
  218. tmp = ktime_set((long)nsec, rem);
  219. }
  220. return ktime_add(kt, tmp);
  221. }
  222. #else /* CONFIG_KTIME_SCALAR */
  223. # endif /* !CONFIG_KTIME_SCALAR */
  224. /*
  225. * Divide a ktime value by a nanosecond value
  226. */
  227. static unsigned long ktime_divns(const ktime_t kt, s64 div)
  228. {
  229. u64 dclc, inc, dns;
  230. int sft = 0;
  231. dclc = dns = ktime_to_ns(kt);
  232. inc = div;
  233. /* Make sure the divisor is less than 2^32: */
  234. while (div >> 32) {
  235. sft++;
  236. div >>= 1;
  237. }
  238. dclc >>= sft;
  239. do_div(dclc, (unsigned long) div);
  240. return (unsigned long) dclc;
  241. }
  242. #else /* BITS_PER_LONG < 64 */
  243. # define ktime_divns(kt, div) (unsigned long)((kt).tv64 / (div))
  244. #endif /* BITS_PER_LONG >= 64 */
  245. /*
  246. * Counterpart to lock_timer_base above:
  247. */
  248. static inline
  249. void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
  250. {
  251. spin_unlock_irqrestore(&timer->base->lock, *flags);
  252. }
  253. /**
  254. * hrtimer_forward - forward the timer expiry
  255. *
  256. * @timer: hrtimer to forward
  257. * @now: forward past this time
  258. * @interval: the interval to forward
  259. *
  260. * Forward the timer expiry so it will expire in the future.
  261. * Returns the number of overruns.
  262. */
  263. unsigned long
  264. hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
  265. {
  266. unsigned long orun = 1;
  267. ktime_t delta;
  268. delta = ktime_sub(now, timer->expires);
  269. if (delta.tv64 < 0)
  270. return 0;
  271. if (interval.tv64 < timer->base->resolution.tv64)
  272. interval.tv64 = timer->base->resolution.tv64;
  273. if (unlikely(delta.tv64 >= interval.tv64)) {
  274. s64 incr = ktime_to_ns(interval);
  275. orun = ktime_divns(delta, incr);
  276. timer->expires = ktime_add_ns(timer->expires, incr * orun);
  277. if (timer->expires.tv64 > now.tv64)
  278. return orun;
  279. /*
  280. * This (and the ktime_add() below) is the
  281. * correction for exact:
  282. */
  283. orun++;
  284. }
  285. timer->expires = ktime_add(timer->expires, interval);
  286. return orun;
  287. }
  288. /*
  289. * enqueue_hrtimer - internal function to (re)start a timer
  290. *
  291. * The timer is inserted in expiry order. Insertion into the
  292. * red black tree is O(log(n)). Must hold the base lock.
  293. */
  294. static void enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
  295. {
  296. struct rb_node **link = &base->active.rb_node;
  297. struct rb_node *parent = NULL;
  298. struct hrtimer *entry;
  299. /*
  300. * Find the right place in the rbtree:
  301. */
  302. while (*link) {
  303. parent = *link;
  304. entry = rb_entry(parent, struct hrtimer, node);
  305. /*
  306. * We dont care about collisions. Nodes with
  307. * the same expiry time stay together.
  308. */
  309. if (timer->expires.tv64 < entry->expires.tv64)
  310. link = &(*link)->rb_left;
  311. else
  312. link = &(*link)->rb_right;
  313. }
  314. /*
  315. * Insert the timer to the rbtree and check whether it
  316. * replaces the first pending timer
  317. */
  318. rb_link_node(&timer->node, parent, link);
  319. rb_insert_color(&timer->node, &base->active);
  320. if (!base->first || timer->expires.tv64 <
  321. rb_entry(base->first, struct hrtimer, node)->expires.tv64)
  322. base->first = &timer->node;
  323. }
  324. /*
  325. * __remove_hrtimer - internal function to remove a timer
  326. *
  327. * Caller must hold the base lock.
  328. */
  329. static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
  330. {
  331. /*
  332. * Remove the timer from the rbtree and replace the
  333. * first entry pointer if necessary.
  334. */
  335. if (base->first == &timer->node)
  336. base->first = rb_next(&timer->node);
  337. rb_erase(&timer->node, &base->active);
  338. timer->node.rb_parent = HRTIMER_INACTIVE;
  339. }
  340. /*
  341. * remove hrtimer, called with base lock held
  342. */
  343. static inline int
  344. remove_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
  345. {
  346. if (hrtimer_active(timer)) {
  347. __remove_hrtimer(timer, base);
  348. return 1;
  349. }
  350. return 0;
  351. }
  352. /**
  353. * hrtimer_start - (re)start an relative timer on the current CPU
  354. *
  355. * @timer: the timer to be added
  356. * @tim: expiry time
  357. * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
  358. *
  359. * Returns:
  360. * 0 on success
  361. * 1 when the timer was active
  362. */
  363. int
  364. hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
  365. {
  366. struct hrtimer_base *base, *new_base;
  367. unsigned long flags;
  368. int ret;
  369. base = lock_hrtimer_base(timer, &flags);
  370. /* Remove an active timer from the queue: */
  371. ret = remove_hrtimer(timer, base);
  372. /* Switch the timer base, if necessary: */
  373. new_base = switch_hrtimer_base(timer, base);
  374. if (mode == HRTIMER_REL) {
  375. tim = ktime_add(tim, new_base->get_time());
  376. /*
  377. * CONFIG_TIME_LOW_RES is a temporary way for architectures
  378. * to signal that they simply return xtime in
  379. * do_gettimeoffset(). In this case we want to round up by
  380. * resolution when starting a relative timer, to avoid short
  381. * timeouts. This will go away with the GTOD framework.
  382. */
  383. #ifdef CONFIG_TIME_LOW_RES
  384. tim = ktime_add(tim, base->resolution);
  385. #endif
  386. }
  387. timer->expires = tim;
  388. enqueue_hrtimer(timer, new_base);
  389. unlock_hrtimer_base(timer, &flags);
  390. return ret;
  391. }
  392. /**
  393. * hrtimer_try_to_cancel - try to deactivate a timer
  394. *
  395. * @timer: hrtimer to stop
  396. *
  397. * Returns:
  398. * 0 when the timer was not active
  399. * 1 when the timer was active
  400. * -1 when the timer is currently excuting the callback function and
  401. * can not be stopped
  402. */
  403. int hrtimer_try_to_cancel(struct hrtimer *timer)
  404. {
  405. struct hrtimer_base *base;
  406. unsigned long flags;
  407. int ret = -1;
  408. base = lock_hrtimer_base(timer, &flags);
  409. if (base->curr_timer != timer)
  410. ret = remove_hrtimer(timer, base);
  411. unlock_hrtimer_base(timer, &flags);
  412. return ret;
  413. }
  414. /**
  415. * hrtimer_cancel - cancel a timer and wait for the handler to finish.
  416. *
  417. * @timer: the timer to be cancelled
  418. *
  419. * Returns:
  420. * 0 when the timer was not active
  421. * 1 when the timer was active
  422. */
  423. int hrtimer_cancel(struct hrtimer *timer)
  424. {
  425. for (;;) {
  426. int ret = hrtimer_try_to_cancel(timer);
  427. if (ret >= 0)
  428. return ret;
  429. }
  430. }
  431. /**
  432. * hrtimer_get_remaining - get remaining time for the timer
  433. *
  434. * @timer: the timer to read
  435. */
  436. ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
  437. {
  438. struct hrtimer_base *base;
  439. unsigned long flags;
  440. ktime_t rem;
  441. base = lock_hrtimer_base(timer, &flags);
  442. rem = ktime_sub(timer->expires, timer->base->get_time());
  443. unlock_hrtimer_base(timer, &flags);
  444. return rem;
  445. }
  446. #ifdef CONFIG_NO_IDLE_HZ
  447. /**
  448. * hrtimer_get_next_event - get the time until next expiry event
  449. *
  450. * Returns the delta to the next expiry event or KTIME_MAX if no timer
  451. * is pending.
  452. */
  453. ktime_t hrtimer_get_next_event(void)
  454. {
  455. struct hrtimer_base *base = __get_cpu_var(hrtimer_bases);
  456. ktime_t delta, mindelta = { .tv64 = KTIME_MAX };
  457. unsigned long flags;
  458. int i;
  459. for (i = 0; i < MAX_HRTIMER_BASES; i++, base++) {
  460. struct hrtimer *timer;
  461. spin_lock_irqsave(&base->lock, flags);
  462. if (!base->first) {
  463. spin_unlock_irqrestore(&base->lock, flags);
  464. continue;
  465. }
  466. timer = rb_entry(base->first, struct hrtimer, node);
  467. delta.tv64 = timer->expires.tv64;
  468. spin_unlock_irqrestore(&base->lock, flags);
  469. delta = ktime_sub(delta, base->get_time());
  470. if (delta.tv64 < mindelta.tv64)
  471. mindelta.tv64 = delta.tv64;
  472. }
  473. if (mindelta.tv64 < 0)
  474. mindelta.tv64 = 0;
  475. return mindelta;
  476. }
  477. #endif
  478. /**
  479. * hrtimer_init - initialize a timer to the given clock
  480. *
  481. * @timer: the timer to be initialized
  482. * @clock_id: the clock to be used
  483. * @mode: timer mode abs/rel
  484. */
  485. void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
  486. enum hrtimer_mode mode)
  487. {
  488. struct hrtimer_base *bases;
  489. memset(timer, 0, sizeof(struct hrtimer));
  490. bases = per_cpu(hrtimer_bases, raw_smp_processor_id());
  491. if (clock_id == CLOCK_REALTIME && mode != HRTIMER_ABS)
  492. clock_id = CLOCK_MONOTONIC;
  493. timer->base = &bases[clock_id];
  494. timer->node.rb_parent = HRTIMER_INACTIVE;
  495. }
  496. /**
  497. * hrtimer_get_res - get the timer resolution for a clock
  498. *
  499. * @which_clock: which clock to query
  500. * @tp: pointer to timespec variable to store the resolution
  501. *
  502. * Store the resolution of the clock selected by which_clock in the
  503. * variable pointed to by tp.
  504. */
  505. int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
  506. {
  507. struct hrtimer_base *bases;
  508. bases = per_cpu(hrtimer_bases, raw_smp_processor_id());
  509. *tp = ktime_to_timespec(bases[which_clock].resolution);
  510. return 0;
  511. }
  512. /*
  513. * Expire the per base hrtimer-queue:
  514. */
  515. static inline void run_hrtimer_queue(struct hrtimer_base *base)
  516. {
  517. struct rb_node *node;
  518. if (base->get_softirq_time)
  519. base->softirq_time = base->get_softirq_time();
  520. spin_lock_irq(&base->lock);
  521. while ((node = base->first)) {
  522. struct hrtimer *timer;
  523. int (*fn)(struct hrtimer *);
  524. int restart;
  525. timer = rb_entry(node, struct hrtimer, node);
  526. if (base->softirq_time.tv64 <= timer->expires.tv64)
  527. break;
  528. fn = timer->function;
  529. set_curr_timer(base, timer);
  530. __remove_hrtimer(timer, base);
  531. spin_unlock_irq(&base->lock);
  532. restart = fn(timer);
  533. spin_lock_irq(&base->lock);
  534. if (restart != HRTIMER_NORESTART) {
  535. BUG_ON(hrtimer_active(timer));
  536. enqueue_hrtimer(timer, base);
  537. }
  538. }
  539. set_curr_timer(base, NULL);
  540. spin_unlock_irq(&base->lock);
  541. }
  542. /*
  543. * Called from timer softirq every jiffy, expire hrtimers:
  544. */
  545. void hrtimer_run_queues(void)
  546. {
  547. struct hrtimer_base *base = __get_cpu_var(hrtimer_bases);
  548. int i;
  549. hrtimer_get_softirq_time(base);
  550. for (i = 0; i < MAX_HRTIMER_BASES; i++)
  551. run_hrtimer_queue(&base[i]);
  552. }
  553. /*
  554. * Sleep related functions:
  555. */
  556. struct sleep_hrtimer {
  557. struct hrtimer timer;
  558. struct task_struct *task;
  559. int expired;
  560. };
  561. static int nanosleep_wakeup(struct hrtimer *timer)
  562. {
  563. struct sleep_hrtimer *t =
  564. container_of(timer, struct sleep_hrtimer, timer);
  565. t->expired = 1;
  566. wake_up_process(t->task);
  567. return HRTIMER_NORESTART;
  568. }
  569. static int __sched do_nanosleep(struct sleep_hrtimer *t, enum hrtimer_mode mode)
  570. {
  571. t->timer.function = nanosleep_wakeup;
  572. t->task = current;
  573. t->expired = 0;
  574. do {
  575. set_current_state(TASK_INTERRUPTIBLE);
  576. hrtimer_start(&t->timer, t->timer.expires, mode);
  577. schedule();
  578. if (unlikely(!t->expired)) {
  579. hrtimer_cancel(&t->timer);
  580. mode = HRTIMER_ABS;
  581. }
  582. } while (!t->expired && !signal_pending(current));
  583. return t->expired;
  584. }
  585. static long __sched nanosleep_restart(struct restart_block *restart)
  586. {
  587. struct sleep_hrtimer t;
  588. struct timespec __user *rmtp;
  589. struct timespec tu;
  590. ktime_t time;
  591. restart->fn = do_no_restart_syscall;
  592. hrtimer_init(&t.timer, restart->arg3, HRTIMER_ABS);
  593. t.timer.expires.tv64 = ((u64)restart->arg1 << 32) | (u64) restart->arg0;
  594. if (do_nanosleep(&t, HRTIMER_ABS))
  595. return 0;
  596. rmtp = (struct timespec __user *) restart->arg2;
  597. if (rmtp) {
  598. time = ktime_sub(t.timer.expires, t.timer.base->get_time());
  599. if (time.tv64 <= 0)
  600. return 0;
  601. tu = ktime_to_timespec(time);
  602. if (copy_to_user(rmtp, &tu, sizeof(tu)))
  603. return -EFAULT;
  604. }
  605. restart->fn = nanosleep_restart;
  606. /* The other values in restart are already filled in */
  607. return -ERESTART_RESTARTBLOCK;
  608. }
  609. long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
  610. const enum hrtimer_mode mode, const clockid_t clockid)
  611. {
  612. struct restart_block *restart;
  613. struct sleep_hrtimer t;
  614. struct timespec tu;
  615. ktime_t rem;
  616. hrtimer_init(&t.timer, clockid, mode);
  617. t.timer.expires = timespec_to_ktime(*rqtp);
  618. if (do_nanosleep(&t, mode))
  619. return 0;
  620. /* Absolute timers do not update the rmtp value and restart: */
  621. if (mode == HRTIMER_ABS)
  622. return -ERESTARTNOHAND;
  623. if (rmtp) {
  624. rem = ktime_sub(t.timer.expires, t.timer.base->get_time());
  625. if (rem.tv64 <= 0)
  626. return 0;
  627. tu = ktime_to_timespec(rem);
  628. if (copy_to_user(rmtp, &tu, sizeof(tu)))
  629. return -EFAULT;
  630. }
  631. restart = &current_thread_info()->restart_block;
  632. restart->fn = nanosleep_restart;
  633. restart->arg0 = t.timer.expires.tv64 & 0xFFFFFFFF;
  634. restart->arg1 = t.timer.expires.tv64 >> 32;
  635. restart->arg2 = (unsigned long) rmtp;
  636. restart->arg3 = (unsigned long) t.timer.base->index;
  637. return -ERESTART_RESTARTBLOCK;
  638. }
  639. asmlinkage long
  640. sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp)
  641. {
  642. struct timespec tu;
  643. if (copy_from_user(&tu, rqtp, sizeof(tu)))
  644. return -EFAULT;
  645. if (!timespec_valid(&tu))
  646. return -EINVAL;
  647. return hrtimer_nanosleep(&tu, rmtp, HRTIMER_REL, CLOCK_MONOTONIC);
  648. }
  649. /*
  650. * Functions related to boot-time initialization:
  651. */
  652. static void __devinit init_hrtimers_cpu(int cpu)
  653. {
  654. struct hrtimer_base *base = per_cpu(hrtimer_bases, cpu);
  655. int i;
  656. for (i = 0; i < MAX_HRTIMER_BASES; i++, base++)
  657. spin_lock_init(&base->lock);
  658. }
  659. #ifdef CONFIG_HOTPLUG_CPU
  660. static void migrate_hrtimer_list(struct hrtimer_base *old_base,
  661. struct hrtimer_base *new_base)
  662. {
  663. struct hrtimer *timer;
  664. struct rb_node *node;
  665. while ((node = rb_first(&old_base->active))) {
  666. timer = rb_entry(node, struct hrtimer, node);
  667. __remove_hrtimer(timer, old_base);
  668. timer->base = new_base;
  669. enqueue_hrtimer(timer, new_base);
  670. }
  671. }
  672. static void migrate_hrtimers(int cpu)
  673. {
  674. struct hrtimer_base *old_base, *new_base;
  675. int i;
  676. BUG_ON(cpu_online(cpu));
  677. old_base = per_cpu(hrtimer_bases, cpu);
  678. new_base = get_cpu_var(hrtimer_bases);
  679. local_irq_disable();
  680. for (i = 0; i < MAX_HRTIMER_BASES; i++) {
  681. spin_lock(&new_base->lock);
  682. spin_lock(&old_base->lock);
  683. BUG_ON(old_base->curr_timer);
  684. migrate_hrtimer_list(old_base, new_base);
  685. spin_unlock(&old_base->lock);
  686. spin_unlock(&new_base->lock);
  687. old_base++;
  688. new_base++;
  689. }
  690. local_irq_enable();
  691. put_cpu_var(hrtimer_bases);
  692. }
  693. #endif /* CONFIG_HOTPLUG_CPU */
  694. static int __devinit hrtimer_cpu_notify(struct notifier_block *self,
  695. unsigned long action, void *hcpu)
  696. {
  697. long cpu = (long)hcpu;
  698. switch (action) {
  699. case CPU_UP_PREPARE:
  700. init_hrtimers_cpu(cpu);
  701. break;
  702. #ifdef CONFIG_HOTPLUG_CPU
  703. case CPU_DEAD:
  704. migrate_hrtimers(cpu);
  705. break;
  706. #endif
  707. default:
  708. break;
  709. }
  710. return NOTIFY_OK;
  711. }
  712. static struct notifier_block __devinitdata hrtimers_nb = {
  713. .notifier_call = hrtimer_cpu_notify,
  714. };
  715. void __init hrtimers_init(void)
  716. {
  717. hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE,
  718. (void *)(long)smp_processor_id());
  719. register_cpu_notifier(&hrtimers_nb);
  720. }