hrtimer.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873
  1. /*
  2. * linux/kernel/hrtimer.c
  3. *
  4. * Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright(C) 2005, Red Hat, Inc., Ingo Molnar
  6. *
  7. * High-resolution kernel timers
  8. *
  9. * In contrast to the low-resolution timeout API implemented in
  10. * kernel/timer.c, hrtimers provide finer resolution and accuracy
  11. * depending on system configuration and capabilities.
  12. *
  13. * These timers are currently used for:
  14. * - itimers
  15. * - POSIX timers
  16. * - nanosleep
  17. * - precise in-kernel timing
  18. *
  19. * Started by: Thomas Gleixner and Ingo Molnar
  20. *
  21. * Credits:
  22. * based on kernel/timer.c
  23. *
  24. * Help, testing, suggestions, bugfixes, improvements were
  25. * provided by:
  26. *
  27. * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
  28. * et. al.
  29. *
  30. * For licencing details see kernel-base/COPYING
  31. */
  32. #include <linux/cpu.h>
  33. #include <linux/module.h>
  34. #include <linux/percpu.h>
  35. #include <linux/hrtimer.h>
  36. #include <linux/notifier.h>
  37. #include <linux/syscalls.h>
  38. #include <linux/interrupt.h>
  39. #include <asm/uaccess.h>
  40. /**
  41. * ktime_get - get the monotonic time in ktime_t format
  42. *
  43. * returns the time in ktime_t format
  44. */
  45. static ktime_t ktime_get(void)
  46. {
  47. struct timespec now;
  48. ktime_get_ts(&now);
  49. return timespec_to_ktime(now);
  50. }
  51. /**
  52. * ktime_get_real - get the real (wall-) time in ktime_t format
  53. *
  54. * returns the time in ktime_t format
  55. */
  56. static ktime_t ktime_get_real(void)
  57. {
  58. struct timespec now;
  59. getnstimeofday(&now);
  60. return timespec_to_ktime(now);
  61. }
  62. EXPORT_SYMBOL_GPL(ktime_get_real);
  63. /*
  64. * The timer bases:
  65. *
  66. * Note: If we want to add new timer bases, we have to skip the two
  67. * clock ids captured by the cpu-timers. We do this by holding empty
  68. * entries rather than doing math adjustment of the clock ids.
  69. * This ensures that we capture erroneous accesses to these clock ids
  70. * rather than moving them into the range of valid clock id's.
  71. */
  72. #define MAX_HRTIMER_BASES 2
  73. static DEFINE_PER_CPU(struct hrtimer_base, hrtimer_bases[MAX_HRTIMER_BASES]) =
  74. {
  75. {
  76. .index = CLOCK_REALTIME,
  77. .get_time = &ktime_get_real,
  78. .resolution = KTIME_REALTIME_RES,
  79. },
  80. {
  81. .index = CLOCK_MONOTONIC,
  82. .get_time = &ktime_get,
  83. .resolution = KTIME_MONOTONIC_RES,
  84. },
  85. };
  86. /**
  87. * ktime_get_ts - get the monotonic clock in timespec format
  88. *
  89. * @ts: pointer to timespec variable
  90. *
  91. * The function calculates the monotonic clock from the realtime
  92. * clock and the wall_to_monotonic offset and stores the result
  93. * in normalized timespec format in the variable pointed to by ts.
  94. */
  95. void ktime_get_ts(struct timespec *ts)
  96. {
  97. struct timespec tomono;
  98. unsigned long seq;
  99. do {
  100. seq = read_seqbegin(&xtime_lock);
  101. getnstimeofday(ts);
  102. tomono = wall_to_monotonic;
  103. } while (read_seqretry(&xtime_lock, seq));
  104. set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
  105. ts->tv_nsec + tomono.tv_nsec);
  106. }
  107. EXPORT_SYMBOL_GPL(ktime_get_ts);
  108. /*
  109. * Get the coarse grained time at the softirq based on xtime and
  110. * wall_to_monotonic.
  111. */
  112. static void hrtimer_get_softirq_time(struct hrtimer_base *base)
  113. {
  114. ktime_t xtim, tomono;
  115. unsigned long seq;
  116. do {
  117. seq = read_seqbegin(&xtime_lock);
  118. xtim = timespec_to_ktime(xtime);
  119. tomono = timespec_to_ktime(wall_to_monotonic);
  120. } while (read_seqretry(&xtime_lock, seq));
  121. base[CLOCK_REALTIME].softirq_time = xtim;
  122. base[CLOCK_MONOTONIC].softirq_time = ktime_add(xtim, tomono);
  123. }
  124. /*
  125. * Functions and macros which are different for UP/SMP systems are kept in a
  126. * single place
  127. */
  128. #ifdef CONFIG_SMP
  129. #define set_curr_timer(b, t) do { (b)->curr_timer = (t); } while (0)
  130. /*
  131. * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
  132. * means that all timers which are tied to this base via timer->base are
  133. * locked, and the base itself is locked too.
  134. *
  135. * So __run_timers/migrate_timers can safely modify all timers which could
  136. * be found on the lists/queues.
  137. *
  138. * When the timer's base is locked, and the timer removed from list, it is
  139. * possible to set timer->base = NULL and drop the lock: the timer remains
  140. * locked.
  141. */
  142. static struct hrtimer_base *lock_hrtimer_base(const struct hrtimer *timer,
  143. unsigned long *flags)
  144. {
  145. struct hrtimer_base *base;
  146. for (;;) {
  147. base = timer->base;
  148. if (likely(base != NULL)) {
  149. spin_lock_irqsave(&base->lock, *flags);
  150. if (likely(base == timer->base))
  151. return base;
  152. /* The timer has migrated to another CPU: */
  153. spin_unlock_irqrestore(&base->lock, *flags);
  154. }
  155. cpu_relax();
  156. }
  157. }
  158. /*
  159. * Switch the timer base to the current CPU when possible.
  160. */
  161. static inline struct hrtimer_base *
  162. switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_base *base)
  163. {
  164. struct hrtimer_base *new_base;
  165. new_base = &__get_cpu_var(hrtimer_bases[base->index]);
  166. if (base != new_base) {
  167. /*
  168. * We are trying to schedule the timer on the local CPU.
  169. * However we can't change timer's base while it is running,
  170. * so we keep it on the same CPU. No hassle vs. reprogramming
  171. * the event source in the high resolution case. The softirq
  172. * code will take care of this when the timer function has
  173. * completed. There is no conflict as we hold the lock until
  174. * the timer is enqueued.
  175. */
  176. if (unlikely(base->curr_timer == timer))
  177. return base;
  178. /* See the comment in lock_timer_base() */
  179. timer->base = NULL;
  180. spin_unlock(&base->lock);
  181. spin_lock(&new_base->lock);
  182. timer->base = new_base;
  183. }
  184. return new_base;
  185. }
  186. #else /* CONFIG_SMP */
  187. #define set_curr_timer(b, t) do { } while (0)
  188. static inline struct hrtimer_base *
  189. lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
  190. {
  191. struct hrtimer_base *base = timer->base;
  192. spin_lock_irqsave(&base->lock, *flags);
  193. return base;
  194. }
  195. #define switch_hrtimer_base(t, b) (b)
  196. #endif /* !CONFIG_SMP */
  197. /*
  198. * Functions for the union type storage format of ktime_t which are
  199. * too large for inlining:
  200. */
  201. #if BITS_PER_LONG < 64
  202. # ifndef CONFIG_KTIME_SCALAR
  203. /**
  204. * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable
  205. *
  206. * @kt: addend
  207. * @nsec: the scalar nsec value to add
  208. *
  209. * Returns the sum of kt and nsec in ktime_t format
  210. */
  211. ktime_t ktime_add_ns(const ktime_t kt, u64 nsec)
  212. {
  213. ktime_t tmp;
  214. if (likely(nsec < NSEC_PER_SEC)) {
  215. tmp.tv64 = nsec;
  216. } else {
  217. unsigned long rem = do_div(nsec, NSEC_PER_SEC);
  218. tmp = ktime_set((long)nsec, rem);
  219. }
  220. return ktime_add(kt, tmp);
  221. }
  222. #else /* CONFIG_KTIME_SCALAR */
  223. # endif /* !CONFIG_KTIME_SCALAR */
  224. /*
  225. * Divide a ktime value by a nanosecond value
  226. */
  227. static unsigned long ktime_divns(const ktime_t kt, s64 div)
  228. {
  229. u64 dclc, inc, dns;
  230. int sft = 0;
  231. dclc = dns = ktime_to_ns(kt);
  232. inc = div;
  233. /* Make sure the divisor is less than 2^32: */
  234. while (div >> 32) {
  235. sft++;
  236. div >>= 1;
  237. }
  238. dclc >>= sft;
  239. do_div(dclc, (unsigned long) div);
  240. return (unsigned long) dclc;
  241. }
  242. #else /* BITS_PER_LONG < 64 */
  243. # define ktime_divns(kt, div) (unsigned long)((kt).tv64 / (div))
  244. #endif /* BITS_PER_LONG >= 64 */
  245. /*
  246. * Counterpart to lock_timer_base above:
  247. */
  248. static inline
  249. void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
  250. {
  251. spin_unlock_irqrestore(&timer->base->lock, *flags);
  252. }
  253. /**
  254. * hrtimer_forward - forward the timer expiry
  255. *
  256. * @timer: hrtimer to forward
  257. * @now: forward past this time
  258. * @interval: the interval to forward
  259. *
  260. * Forward the timer expiry so it will expire in the future.
  261. * Returns the number of overruns.
  262. */
  263. unsigned long
  264. hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
  265. {
  266. unsigned long orun = 1;
  267. ktime_t delta;
  268. delta = ktime_sub(now, timer->expires);
  269. if (delta.tv64 < 0)
  270. return 0;
  271. if (interval.tv64 < timer->base->resolution.tv64)
  272. interval.tv64 = timer->base->resolution.tv64;
  273. if (unlikely(delta.tv64 >= interval.tv64)) {
  274. s64 incr = ktime_to_ns(interval);
  275. orun = ktime_divns(delta, incr);
  276. timer->expires = ktime_add_ns(timer->expires, incr * orun);
  277. if (timer->expires.tv64 > now.tv64)
  278. return orun;
  279. /*
  280. * This (and the ktime_add() below) is the
  281. * correction for exact:
  282. */
  283. orun++;
  284. }
  285. timer->expires = ktime_add(timer->expires, interval);
  286. return orun;
  287. }
  288. /*
  289. * enqueue_hrtimer - internal function to (re)start a timer
  290. *
  291. * The timer is inserted in expiry order. Insertion into the
  292. * red black tree is O(log(n)). Must hold the base lock.
  293. */
  294. static void enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
  295. {
  296. struct rb_node **link = &base->active.rb_node;
  297. struct rb_node *parent = NULL;
  298. struct hrtimer *entry;
  299. /*
  300. * Find the right place in the rbtree:
  301. */
  302. while (*link) {
  303. parent = *link;
  304. entry = rb_entry(parent, struct hrtimer, node);
  305. /*
  306. * We dont care about collisions. Nodes with
  307. * the same expiry time stay together.
  308. */
  309. if (timer->expires.tv64 < entry->expires.tv64)
  310. link = &(*link)->rb_left;
  311. else
  312. link = &(*link)->rb_right;
  313. }
  314. /*
  315. * Insert the timer to the rbtree and check whether it
  316. * replaces the first pending timer
  317. */
  318. rb_link_node(&timer->node, parent, link);
  319. rb_insert_color(&timer->node, &base->active);
  320. if (!base->first || timer->expires.tv64 <
  321. rb_entry(base->first, struct hrtimer, node)->expires.tv64)
  322. base->first = &timer->node;
  323. }
  324. /*
  325. * __remove_hrtimer - internal function to remove a timer
  326. *
  327. * Caller must hold the base lock.
  328. */
  329. static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
  330. {
  331. /*
  332. * Remove the timer from the rbtree and replace the
  333. * first entry pointer if necessary.
  334. */
  335. if (base->first == &timer->node)
  336. base->first = rb_next(&timer->node);
  337. rb_erase(&timer->node, &base->active);
  338. timer->node.rb_parent = HRTIMER_INACTIVE;
  339. }
  340. /*
  341. * remove hrtimer, called with base lock held
  342. */
  343. static inline int
  344. remove_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
  345. {
  346. if (hrtimer_active(timer)) {
  347. __remove_hrtimer(timer, base);
  348. return 1;
  349. }
  350. return 0;
  351. }
  352. /**
  353. * hrtimer_start - (re)start an relative timer on the current CPU
  354. *
  355. * @timer: the timer to be added
  356. * @tim: expiry time
  357. * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
  358. *
  359. * Returns:
  360. * 0 on success
  361. * 1 when the timer was active
  362. */
  363. int
  364. hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
  365. {
  366. struct hrtimer_base *base, *new_base;
  367. unsigned long flags;
  368. int ret;
  369. base = lock_hrtimer_base(timer, &flags);
  370. /* Remove an active timer from the queue: */
  371. ret = remove_hrtimer(timer, base);
  372. /* Switch the timer base, if necessary: */
  373. new_base = switch_hrtimer_base(timer, base);
  374. if (mode == HRTIMER_REL) {
  375. tim = ktime_add(tim, new_base->get_time());
  376. /*
  377. * CONFIG_TIME_LOW_RES is a temporary way for architectures
  378. * to signal that they simply return xtime in
  379. * do_gettimeoffset(). In this case we want to round up by
  380. * resolution when starting a relative timer, to avoid short
  381. * timeouts. This will go away with the GTOD framework.
  382. */
  383. #ifdef CONFIG_TIME_LOW_RES
  384. tim = ktime_add(tim, base->resolution);
  385. #endif
  386. }
  387. timer->expires = tim;
  388. enqueue_hrtimer(timer, new_base);
  389. unlock_hrtimer_base(timer, &flags);
  390. return ret;
  391. }
  392. /**
  393. * hrtimer_try_to_cancel - try to deactivate a timer
  394. *
  395. * @timer: hrtimer to stop
  396. *
  397. * Returns:
  398. * 0 when the timer was not active
  399. * 1 when the timer was active
  400. * -1 when the timer is currently excuting the callback function and
  401. * can not be stopped
  402. */
  403. int hrtimer_try_to_cancel(struct hrtimer *timer)
  404. {
  405. struct hrtimer_base *base;
  406. unsigned long flags;
  407. int ret = -1;
  408. base = lock_hrtimer_base(timer, &flags);
  409. if (base->curr_timer != timer)
  410. ret = remove_hrtimer(timer, base);
  411. unlock_hrtimer_base(timer, &flags);
  412. return ret;
  413. }
  414. /**
  415. * hrtimer_cancel - cancel a timer and wait for the handler to finish.
  416. *
  417. * @timer: the timer to be cancelled
  418. *
  419. * Returns:
  420. * 0 when the timer was not active
  421. * 1 when the timer was active
  422. */
  423. int hrtimer_cancel(struct hrtimer *timer)
  424. {
  425. for (;;) {
  426. int ret = hrtimer_try_to_cancel(timer);
  427. if (ret >= 0)
  428. return ret;
  429. cpu_relax();
  430. }
  431. }
  432. /**
  433. * hrtimer_get_remaining - get remaining time for the timer
  434. *
  435. * @timer: the timer to read
  436. */
  437. ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
  438. {
  439. struct hrtimer_base *base;
  440. unsigned long flags;
  441. ktime_t rem;
  442. base = lock_hrtimer_base(timer, &flags);
  443. rem = ktime_sub(timer->expires, timer->base->get_time());
  444. unlock_hrtimer_base(timer, &flags);
  445. return rem;
  446. }
  447. #ifdef CONFIG_NO_IDLE_HZ
  448. /**
  449. * hrtimer_get_next_event - get the time until next expiry event
  450. *
  451. * Returns the delta to the next expiry event or KTIME_MAX if no timer
  452. * is pending.
  453. */
  454. ktime_t hrtimer_get_next_event(void)
  455. {
  456. struct hrtimer_base *base = __get_cpu_var(hrtimer_bases);
  457. ktime_t delta, mindelta = { .tv64 = KTIME_MAX };
  458. unsigned long flags;
  459. int i;
  460. for (i = 0; i < MAX_HRTIMER_BASES; i++, base++) {
  461. struct hrtimer *timer;
  462. spin_lock_irqsave(&base->lock, flags);
  463. if (!base->first) {
  464. spin_unlock_irqrestore(&base->lock, flags);
  465. continue;
  466. }
  467. timer = rb_entry(base->first, struct hrtimer, node);
  468. delta.tv64 = timer->expires.tv64;
  469. spin_unlock_irqrestore(&base->lock, flags);
  470. delta = ktime_sub(delta, base->get_time());
  471. if (delta.tv64 < mindelta.tv64)
  472. mindelta.tv64 = delta.tv64;
  473. }
  474. if (mindelta.tv64 < 0)
  475. mindelta.tv64 = 0;
  476. return mindelta;
  477. }
  478. #endif
  479. /**
  480. * hrtimer_init - initialize a timer to the given clock
  481. *
  482. * @timer: the timer to be initialized
  483. * @clock_id: the clock to be used
  484. * @mode: timer mode abs/rel
  485. */
  486. void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
  487. enum hrtimer_mode mode)
  488. {
  489. struct hrtimer_base *bases;
  490. memset(timer, 0, sizeof(struct hrtimer));
  491. bases = per_cpu(hrtimer_bases, raw_smp_processor_id());
  492. if (clock_id == CLOCK_REALTIME && mode != HRTIMER_ABS)
  493. clock_id = CLOCK_MONOTONIC;
  494. timer->base = &bases[clock_id];
  495. timer->node.rb_parent = HRTIMER_INACTIVE;
  496. }
  497. /**
  498. * hrtimer_get_res - get the timer resolution for a clock
  499. *
  500. * @which_clock: which clock to query
  501. * @tp: pointer to timespec variable to store the resolution
  502. *
  503. * Store the resolution of the clock selected by which_clock in the
  504. * variable pointed to by tp.
  505. */
  506. int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
  507. {
  508. struct hrtimer_base *bases;
  509. bases = per_cpu(hrtimer_bases, raw_smp_processor_id());
  510. *tp = ktime_to_timespec(bases[which_clock].resolution);
  511. return 0;
  512. }
  513. /*
  514. * Expire the per base hrtimer-queue:
  515. */
  516. static inline void run_hrtimer_queue(struct hrtimer_base *base)
  517. {
  518. struct rb_node *node;
  519. if (!base->first)
  520. return;
  521. if (base->get_softirq_time)
  522. base->softirq_time = base->get_softirq_time();
  523. spin_lock_irq(&base->lock);
  524. while ((node = base->first)) {
  525. struct hrtimer *timer;
  526. int (*fn)(struct hrtimer *);
  527. int restart;
  528. timer = rb_entry(node, struct hrtimer, node);
  529. if (base->softirq_time.tv64 <= timer->expires.tv64)
  530. break;
  531. fn = timer->function;
  532. set_curr_timer(base, timer);
  533. __remove_hrtimer(timer, base);
  534. spin_unlock_irq(&base->lock);
  535. restart = fn(timer);
  536. spin_lock_irq(&base->lock);
  537. if (restart != HRTIMER_NORESTART) {
  538. BUG_ON(hrtimer_active(timer));
  539. enqueue_hrtimer(timer, base);
  540. }
  541. }
  542. set_curr_timer(base, NULL);
  543. spin_unlock_irq(&base->lock);
  544. }
  545. /*
  546. * Called from timer softirq every jiffy, expire hrtimers:
  547. */
  548. void hrtimer_run_queues(void)
  549. {
  550. struct hrtimer_base *base = __get_cpu_var(hrtimer_bases);
  551. int i;
  552. hrtimer_get_softirq_time(base);
  553. for (i = 0; i < MAX_HRTIMER_BASES; i++)
  554. run_hrtimer_queue(&base[i]);
  555. }
  556. /*
  557. * Sleep related functions:
  558. */
  559. static int hrtimer_wakeup(struct hrtimer *timer)
  560. {
  561. struct hrtimer_sleeper *t =
  562. container_of(timer, struct hrtimer_sleeper, timer);
  563. struct task_struct *task = t->task;
  564. t->task = NULL;
  565. if (task)
  566. wake_up_process(task);
  567. return HRTIMER_NORESTART;
  568. }
  569. void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, task_t *task)
  570. {
  571. sl->timer.function = hrtimer_wakeup;
  572. sl->task = task;
  573. }
  574. static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
  575. {
  576. hrtimer_init_sleeper(t, current);
  577. do {
  578. set_current_state(TASK_INTERRUPTIBLE);
  579. hrtimer_start(&t->timer, t->timer.expires, mode);
  580. schedule();
  581. hrtimer_cancel(&t->timer);
  582. mode = HRTIMER_ABS;
  583. } while (t->task && !signal_pending(current));
  584. return t->task == NULL;
  585. }
  586. static long __sched nanosleep_restart(struct restart_block *restart)
  587. {
  588. struct hrtimer_sleeper t;
  589. struct timespec __user *rmtp;
  590. struct timespec tu;
  591. ktime_t time;
  592. restart->fn = do_no_restart_syscall;
  593. hrtimer_init(&t.timer, restart->arg3, HRTIMER_ABS);
  594. t.timer.expires.tv64 = ((u64)restart->arg1 << 32) | (u64) restart->arg0;
  595. if (do_nanosleep(&t, HRTIMER_ABS))
  596. return 0;
  597. rmtp = (struct timespec __user *) restart->arg2;
  598. if (rmtp) {
  599. time = ktime_sub(t.timer.expires, t.timer.base->get_time());
  600. if (time.tv64 <= 0)
  601. return 0;
  602. tu = ktime_to_timespec(time);
  603. if (copy_to_user(rmtp, &tu, sizeof(tu)))
  604. return -EFAULT;
  605. }
  606. restart->fn = nanosleep_restart;
  607. /* The other values in restart are already filled in */
  608. return -ERESTART_RESTARTBLOCK;
  609. }
  610. long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
  611. const enum hrtimer_mode mode, const clockid_t clockid)
  612. {
  613. struct restart_block *restart;
  614. struct hrtimer_sleeper t;
  615. struct timespec tu;
  616. ktime_t rem;
  617. hrtimer_init(&t.timer, clockid, mode);
  618. t.timer.expires = timespec_to_ktime(*rqtp);
  619. if (do_nanosleep(&t, mode))
  620. return 0;
  621. /* Absolute timers do not update the rmtp value and restart: */
  622. if (mode == HRTIMER_ABS)
  623. return -ERESTARTNOHAND;
  624. if (rmtp) {
  625. rem = ktime_sub(t.timer.expires, t.timer.base->get_time());
  626. if (rem.tv64 <= 0)
  627. return 0;
  628. tu = ktime_to_timespec(rem);
  629. if (copy_to_user(rmtp, &tu, sizeof(tu)))
  630. return -EFAULT;
  631. }
  632. restart = &current_thread_info()->restart_block;
  633. restart->fn = nanosleep_restart;
  634. restart->arg0 = t.timer.expires.tv64 & 0xFFFFFFFF;
  635. restart->arg1 = t.timer.expires.tv64 >> 32;
  636. restart->arg2 = (unsigned long) rmtp;
  637. restart->arg3 = (unsigned long) t.timer.base->index;
  638. return -ERESTART_RESTARTBLOCK;
  639. }
  640. asmlinkage long
  641. sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp)
  642. {
  643. struct timespec tu;
  644. if (copy_from_user(&tu, rqtp, sizeof(tu)))
  645. return -EFAULT;
  646. if (!timespec_valid(&tu))
  647. return -EINVAL;
  648. return hrtimer_nanosleep(&tu, rmtp, HRTIMER_REL, CLOCK_MONOTONIC);
  649. }
  650. /*
  651. * Functions related to boot-time initialization:
  652. */
  653. static void __devinit init_hrtimers_cpu(int cpu)
  654. {
  655. struct hrtimer_base *base = per_cpu(hrtimer_bases, cpu);
  656. int i;
  657. for (i = 0; i < MAX_HRTIMER_BASES; i++, base++)
  658. spin_lock_init(&base->lock);
  659. }
  660. #ifdef CONFIG_HOTPLUG_CPU
  661. static void migrate_hrtimer_list(struct hrtimer_base *old_base,
  662. struct hrtimer_base *new_base)
  663. {
  664. struct hrtimer *timer;
  665. struct rb_node *node;
  666. while ((node = rb_first(&old_base->active))) {
  667. timer = rb_entry(node, struct hrtimer, node);
  668. __remove_hrtimer(timer, old_base);
  669. timer->base = new_base;
  670. enqueue_hrtimer(timer, new_base);
  671. }
  672. }
  673. static void migrate_hrtimers(int cpu)
  674. {
  675. struct hrtimer_base *old_base, *new_base;
  676. int i;
  677. BUG_ON(cpu_online(cpu));
  678. old_base = per_cpu(hrtimer_bases, cpu);
  679. new_base = get_cpu_var(hrtimer_bases);
  680. local_irq_disable();
  681. for (i = 0; i < MAX_HRTIMER_BASES; i++) {
  682. spin_lock(&new_base->lock);
  683. spin_lock(&old_base->lock);
  684. BUG_ON(old_base->curr_timer);
  685. migrate_hrtimer_list(old_base, new_base);
  686. spin_unlock(&old_base->lock);
  687. spin_unlock(&new_base->lock);
  688. old_base++;
  689. new_base++;
  690. }
  691. local_irq_enable();
  692. put_cpu_var(hrtimer_bases);
  693. }
  694. #endif /* CONFIG_HOTPLUG_CPU */
  695. static int hrtimer_cpu_notify(struct notifier_block *self,
  696. unsigned long action, void *hcpu)
  697. {
  698. long cpu = (long)hcpu;
  699. switch (action) {
  700. case CPU_UP_PREPARE:
  701. init_hrtimers_cpu(cpu);
  702. break;
  703. #ifdef CONFIG_HOTPLUG_CPU
  704. case CPU_DEAD:
  705. migrate_hrtimers(cpu);
  706. break;
  707. #endif
  708. default:
  709. break;
  710. }
  711. return NOTIFY_OK;
  712. }
  713. static struct notifier_block hrtimers_nb = {
  714. .notifier_call = hrtimer_cpu_notify,
  715. };
  716. void __init hrtimers_init(void)
  717. {
  718. hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE,
  719. (void *)(long)smp_processor_id());
  720. register_cpu_notifier(&hrtimers_nb);
  721. }