timer.c 43 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699
  1. /*
  2. * linux/kernel/timer.c
  3. *
  4. * Kernel internal timers, kernel timekeeping, basic process system calls
  5. *
  6. * Copyright (C) 1991, 1992 Linus Torvalds
  7. *
  8. * 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better.
  9. *
  10. * 1997-09-10 Updated NTP code according to technical memorandum Jan '96
  11. * "A Kernel Model for Precision Timekeeping" by Dave Mills
  12. * 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to
  13. * serialize accesses to xtime/lost_ticks).
  14. * Copyright (C) 1998 Andrea Arcangeli
  15. * 1999-03-10 Improved NTP compatibility by Ulrich Windl
  16. * 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love
  17. * 2000-10-05 Implemented scalable SMP per-CPU timer handling.
  18. * Copyright (C) 2000, 2001, 2002 Ingo Molnar
  19. * Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar
  20. */
  21. #include <linux/kernel_stat.h>
  22. #include <linux/module.h>
  23. #include <linux/interrupt.h>
  24. #include <linux/percpu.h>
  25. #include <linux/init.h>
  26. #include <linux/mm.h>
  27. #include <linux/swap.h>
  28. #include <linux/notifier.h>
  29. #include <linux/thread_info.h>
  30. #include <linux/time.h>
  31. #include <linux/jiffies.h>
  32. #include <linux/posix-timers.h>
  33. #include <linux/cpu.h>
  34. #include <linux/syscalls.h>
  35. #include <linux/delay.h>
  36. #include <asm/uaccess.h>
  37. #include <asm/unistd.h>
  38. #include <asm/div64.h>
  39. #include <asm/timex.h>
  40. #include <asm/io.h>
  41. u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;
  42. EXPORT_SYMBOL(jiffies_64);
  43. /*
  44. * per-CPU timer vector definitions:
  45. */
  46. #define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6)
  47. #define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8)
  48. #define TVN_SIZE (1 << TVN_BITS)
  49. #define TVR_SIZE (1 << TVR_BITS)
  50. #define TVN_MASK (TVN_SIZE - 1)
  51. #define TVR_MASK (TVR_SIZE - 1)
  52. typedef struct tvec_s {
  53. struct list_head vec[TVN_SIZE];
  54. } tvec_t;
  55. typedef struct tvec_root_s {
  56. struct list_head vec[TVR_SIZE];
  57. } tvec_root_t;
  58. struct tvec_t_base_s {
  59. spinlock_t lock;
  60. struct timer_list *running_timer;
  61. unsigned long timer_jiffies;
  62. tvec_root_t tv1;
  63. tvec_t tv2;
  64. tvec_t tv3;
  65. tvec_t tv4;
  66. tvec_t tv5;
  67. } ____cacheline_aligned_in_smp;
  68. typedef struct tvec_t_base_s tvec_base_t;
  69. tvec_base_t boot_tvec_bases;
  70. EXPORT_SYMBOL(boot_tvec_bases);
  71. static DEFINE_PER_CPU(tvec_base_t *, tvec_bases) = &boot_tvec_bases;
  72. static inline void set_running_timer(tvec_base_t *base,
  73. struct timer_list *timer)
  74. {
  75. #ifdef CONFIG_SMP
  76. base->running_timer = timer;
  77. #endif
  78. }
  79. static void internal_add_timer(tvec_base_t *base, struct timer_list *timer)
  80. {
  81. unsigned long expires = timer->expires;
  82. unsigned long idx = expires - base->timer_jiffies;
  83. struct list_head *vec;
  84. if (idx < TVR_SIZE) {
  85. int i = expires & TVR_MASK;
  86. vec = base->tv1.vec + i;
  87. } else if (idx < 1 << (TVR_BITS + TVN_BITS)) {
  88. int i = (expires >> TVR_BITS) & TVN_MASK;
  89. vec = base->tv2.vec + i;
  90. } else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) {
  91. int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK;
  92. vec = base->tv3.vec + i;
  93. } else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) {
  94. int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK;
  95. vec = base->tv4.vec + i;
  96. } else if ((signed long) idx < 0) {
  97. /*
  98. * Can happen if you add a timer with expires == jiffies,
  99. * or you set a timer to go off in the past
  100. */
  101. vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);
  102. } else {
  103. int i;
  104. /* If the timeout is larger than 0xffffffff on 64-bit
  105. * architectures then we use the maximum timeout:
  106. */
  107. if (idx > 0xffffffffUL) {
  108. idx = 0xffffffffUL;
  109. expires = idx + base->timer_jiffies;
  110. }
  111. i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK;
  112. vec = base->tv5.vec + i;
  113. }
  114. /*
  115. * Timers are FIFO:
  116. */
  117. list_add_tail(&timer->entry, vec);
  118. }
  119. /**
  120. * init_timer - initialize a timer.
  121. * @timer: the timer to be initialized
  122. *
  123. * init_timer() must be done to a timer prior calling *any* of the
  124. * other timer functions.
  125. */
  126. void fastcall init_timer(struct timer_list *timer)
  127. {
  128. timer->entry.next = NULL;
  129. timer->base = __raw_get_cpu_var(tvec_bases);
  130. }
  131. EXPORT_SYMBOL(init_timer);
  132. static inline void detach_timer(struct timer_list *timer,
  133. int clear_pending)
  134. {
  135. struct list_head *entry = &timer->entry;
  136. __list_del(entry->prev, entry->next);
  137. if (clear_pending)
  138. entry->next = NULL;
  139. entry->prev = LIST_POISON2;
  140. }
  141. /*
  142. * We are using hashed locking: holding per_cpu(tvec_bases).lock
  143. * means that all timers which are tied to this base via timer->base are
  144. * locked, and the base itself is locked too.
  145. *
  146. * So __run_timers/migrate_timers can safely modify all timers which could
  147. * be found on ->tvX lists.
  148. *
  149. * When the timer's base is locked, and the timer removed from list, it is
  150. * possible to set timer->base = NULL and drop the lock: the timer remains
  151. * locked.
  152. */
  153. static tvec_base_t *lock_timer_base(struct timer_list *timer,
  154. unsigned long *flags)
  155. __acquires(timer->base->lock)
  156. {
  157. tvec_base_t *base;
  158. for (;;) {
  159. base = timer->base;
  160. if (likely(base != NULL)) {
  161. spin_lock_irqsave(&base->lock, *flags);
  162. if (likely(base == timer->base))
  163. return base;
  164. /* The timer has migrated to another CPU */
  165. spin_unlock_irqrestore(&base->lock, *flags);
  166. }
  167. cpu_relax();
  168. }
  169. }
  170. int __mod_timer(struct timer_list *timer, unsigned long expires)
  171. {
  172. tvec_base_t *base, *new_base;
  173. unsigned long flags;
  174. int ret = 0;
  175. BUG_ON(!timer->function);
  176. base = lock_timer_base(timer, &flags);
  177. if (timer_pending(timer)) {
  178. detach_timer(timer, 0);
  179. ret = 1;
  180. }
  181. new_base = __get_cpu_var(tvec_bases);
  182. if (base != new_base) {
  183. /*
  184. * We are trying to schedule the timer on the local CPU.
  185. * However we can't change timer's base while it is running,
  186. * otherwise del_timer_sync() can't detect that the timer's
  187. * handler yet has not finished. This also guarantees that
  188. * the timer is serialized wrt itself.
  189. */
  190. if (likely(base->running_timer != timer)) {
  191. /* See the comment in lock_timer_base() */
  192. timer->base = NULL;
  193. spin_unlock(&base->lock);
  194. base = new_base;
  195. spin_lock(&base->lock);
  196. timer->base = base;
  197. }
  198. }
  199. timer->expires = expires;
  200. internal_add_timer(base, timer);
  201. spin_unlock_irqrestore(&base->lock, flags);
  202. return ret;
  203. }
  204. EXPORT_SYMBOL(__mod_timer);
  205. /**
  206. * add_timer_on - start a timer on a particular CPU
  207. * @timer: the timer to be added
  208. * @cpu: the CPU to start it on
  209. *
  210. * This is not very scalable on SMP. Double adds are not possible.
  211. */
  212. void add_timer_on(struct timer_list *timer, int cpu)
  213. {
  214. tvec_base_t *base = per_cpu(tvec_bases, cpu);
  215. unsigned long flags;
  216. BUG_ON(timer_pending(timer) || !timer->function);
  217. spin_lock_irqsave(&base->lock, flags);
  218. timer->base = base;
  219. internal_add_timer(base, timer);
  220. spin_unlock_irqrestore(&base->lock, flags);
  221. }
  222. /**
  223. * mod_timer - modify a timer's timeout
  224. * @timer: the timer to be modified
  225. * @expires: new timeout in jiffies
  226. *
  227. * mod_timer is a more efficient way to update the expire field of an
  228. * active timer (if the timer is inactive it will be activated)
  229. *
  230. * mod_timer(timer, expires) is equivalent to:
  231. *
  232. * del_timer(timer); timer->expires = expires; add_timer(timer);
  233. *
  234. * Note that if there are multiple unserialized concurrent users of the
  235. * same timer, then mod_timer() is the only safe way to modify the timeout,
  236. * since add_timer() cannot modify an already running timer.
  237. *
  238. * The function returns whether it has modified a pending timer or not.
  239. * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an
  240. * active timer returns 1.)
  241. */
  242. int mod_timer(struct timer_list *timer, unsigned long expires)
  243. {
  244. BUG_ON(!timer->function);
  245. /*
  246. * This is a common optimization triggered by the
  247. * networking code - if the timer is re-modified
  248. * to be the same thing then just return:
  249. */
  250. if (timer->expires == expires && timer_pending(timer))
  251. return 1;
  252. return __mod_timer(timer, expires);
  253. }
  254. EXPORT_SYMBOL(mod_timer);
  255. /**
  256. * del_timer - deactive a timer.
  257. * @timer: the timer to be deactivated
  258. *
  259. * del_timer() deactivates a timer - this works on both active and inactive
  260. * timers.
  261. *
  262. * The function returns whether it has deactivated a pending timer or not.
  263. * (ie. del_timer() of an inactive timer returns 0, del_timer() of an
  264. * active timer returns 1.)
  265. */
  266. int del_timer(struct timer_list *timer)
  267. {
  268. tvec_base_t *base;
  269. unsigned long flags;
  270. int ret = 0;
  271. if (timer_pending(timer)) {
  272. base = lock_timer_base(timer, &flags);
  273. if (timer_pending(timer)) {
  274. detach_timer(timer, 1);
  275. ret = 1;
  276. }
  277. spin_unlock_irqrestore(&base->lock, flags);
  278. }
  279. return ret;
  280. }
  281. EXPORT_SYMBOL(del_timer);
  282. #ifdef CONFIG_SMP
  283. /**
  284. * try_to_del_timer_sync - Try to deactivate a timer
  285. * @timer: timer do del
  286. *
  287. * This function tries to deactivate a timer. Upon successful (ret >= 0)
  288. * exit the timer is not queued and the handler is not running on any CPU.
  289. *
  290. * It must not be called from interrupt contexts.
  291. */
  292. int try_to_del_timer_sync(struct timer_list *timer)
  293. {
  294. tvec_base_t *base;
  295. unsigned long flags;
  296. int ret = -1;
  297. base = lock_timer_base(timer, &flags);
  298. if (base->running_timer == timer)
  299. goto out;
  300. ret = 0;
  301. if (timer_pending(timer)) {
  302. detach_timer(timer, 1);
  303. ret = 1;
  304. }
  305. out:
  306. spin_unlock_irqrestore(&base->lock, flags);
  307. return ret;
  308. }
  309. /**
  310. * del_timer_sync - deactivate a timer and wait for the handler to finish.
  311. * @timer: the timer to be deactivated
  312. *
  313. * This function only differs from del_timer() on SMP: besides deactivating
  314. * the timer it also makes sure the handler has finished executing on other
  315. * CPUs.
  316. *
  317. * Synchronization rules: callers must prevent restarting of the timer,
  318. * otherwise this function is meaningless. It must not be called from
  319. * interrupt contexts. The caller must not hold locks which would prevent
  320. * completion of the timer's handler. The timer's handler must not call
  321. * add_timer_on(). Upon exit the timer is not queued and the handler is
  322. * not running on any CPU.
  323. *
  324. * The function returns whether it has deactivated a pending timer or not.
  325. */
  326. int del_timer_sync(struct timer_list *timer)
  327. {
  328. for (;;) {
  329. int ret = try_to_del_timer_sync(timer);
  330. if (ret >= 0)
  331. return ret;
  332. cpu_relax();
  333. }
  334. }
  335. EXPORT_SYMBOL(del_timer_sync);
  336. #endif
  337. static int cascade(tvec_base_t *base, tvec_t *tv, int index)
  338. {
  339. /* cascade all the timers from tv up one level */
  340. struct timer_list *timer, *tmp;
  341. struct list_head tv_list;
  342. list_replace_init(tv->vec + index, &tv_list);
  343. /*
  344. * We are removing _all_ timers from the list, so we
  345. * don't have to detach them individually.
  346. */
  347. list_for_each_entry_safe(timer, tmp, &tv_list, entry) {
  348. BUG_ON(timer->base != base);
  349. internal_add_timer(base, timer);
  350. }
  351. return index;
  352. }
  353. #define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK)
  354. /**
  355. * __run_timers - run all expired timers (if any) on this CPU.
  356. * @base: the timer vector to be processed.
  357. *
  358. * This function cascades all vectors and executes all expired timer
  359. * vectors.
  360. */
  361. static inline void __run_timers(tvec_base_t *base)
  362. {
  363. struct timer_list *timer;
  364. spin_lock_irq(&base->lock);
  365. while (time_after_eq(jiffies, base->timer_jiffies)) {
  366. struct list_head work_list;
  367. struct list_head *head = &work_list;
  368. int index = base->timer_jiffies & TVR_MASK;
  369. /*
  370. * Cascade timers:
  371. */
  372. if (!index &&
  373. (!cascade(base, &base->tv2, INDEX(0))) &&
  374. (!cascade(base, &base->tv3, INDEX(1))) &&
  375. !cascade(base, &base->tv4, INDEX(2)))
  376. cascade(base, &base->tv5, INDEX(3));
  377. ++base->timer_jiffies;
  378. list_replace_init(base->tv1.vec + index, &work_list);
  379. while (!list_empty(head)) {
  380. void (*fn)(unsigned long);
  381. unsigned long data;
  382. timer = list_entry(head->next,struct timer_list,entry);
  383. fn = timer->function;
  384. data = timer->data;
  385. set_running_timer(base, timer);
  386. detach_timer(timer, 1);
  387. spin_unlock_irq(&base->lock);
  388. {
  389. int preempt_count = preempt_count();
  390. fn(data);
  391. if (preempt_count != preempt_count()) {
  392. printk(KERN_WARNING "huh, entered %p "
  393. "with preempt_count %08x, exited"
  394. " with %08x?\n",
  395. fn, preempt_count,
  396. preempt_count());
  397. BUG();
  398. }
  399. }
  400. spin_lock_irq(&base->lock);
  401. }
  402. }
  403. set_running_timer(base, NULL);
  404. spin_unlock_irq(&base->lock);
  405. }
  406. #ifdef CONFIG_NO_IDLE_HZ
  407. /*
  408. * Find out when the next timer event is due to happen. This
  409. * is used on S/390 to stop all activity when a cpus is idle.
  410. * This functions needs to be called disabled.
  411. */
  412. unsigned long next_timer_interrupt(void)
  413. {
  414. tvec_base_t *base;
  415. struct list_head *list;
  416. struct timer_list *nte;
  417. unsigned long expires;
  418. unsigned long hr_expires = MAX_JIFFY_OFFSET;
  419. ktime_t hr_delta;
  420. tvec_t *varray[4];
  421. int i, j;
  422. hr_delta = hrtimer_get_next_event();
  423. if (hr_delta.tv64 != KTIME_MAX) {
  424. struct timespec tsdelta;
  425. tsdelta = ktime_to_timespec(hr_delta);
  426. hr_expires = timespec_to_jiffies(&tsdelta);
  427. if (hr_expires < 3)
  428. return hr_expires + jiffies;
  429. }
  430. hr_expires += jiffies;
  431. base = __get_cpu_var(tvec_bases);
  432. spin_lock(&base->lock);
  433. expires = base->timer_jiffies + (LONG_MAX >> 1);
  434. list = NULL;
  435. /* Look for timer events in tv1. */
  436. j = base->timer_jiffies & TVR_MASK;
  437. do {
  438. list_for_each_entry(nte, base->tv1.vec + j, entry) {
  439. expires = nte->expires;
  440. if (j < (base->timer_jiffies & TVR_MASK))
  441. list = base->tv2.vec + (INDEX(0));
  442. goto found;
  443. }
  444. j = (j + 1) & TVR_MASK;
  445. } while (j != (base->timer_jiffies & TVR_MASK));
  446. /* Check tv2-tv5. */
  447. varray[0] = &base->tv2;
  448. varray[1] = &base->tv3;
  449. varray[2] = &base->tv4;
  450. varray[3] = &base->tv5;
  451. for (i = 0; i < 4; i++) {
  452. j = INDEX(i);
  453. do {
  454. if (list_empty(varray[i]->vec + j)) {
  455. j = (j + 1) & TVN_MASK;
  456. continue;
  457. }
  458. list_for_each_entry(nte, varray[i]->vec + j, entry)
  459. if (time_before(nte->expires, expires))
  460. expires = nte->expires;
  461. if (j < (INDEX(i)) && i < 3)
  462. list = varray[i + 1]->vec + (INDEX(i + 1));
  463. goto found;
  464. } while (j != (INDEX(i)));
  465. }
  466. found:
  467. if (list) {
  468. /*
  469. * The search wrapped. We need to look at the next list
  470. * from next tv element that would cascade into tv element
  471. * where we found the timer element.
  472. */
  473. list_for_each_entry(nte, list, entry) {
  474. if (time_before(nte->expires, expires))
  475. expires = nte->expires;
  476. }
  477. }
  478. spin_unlock(&base->lock);
  479. /*
  480. * It can happen that other CPUs service timer IRQs and increment
  481. * jiffies, but we have not yet got a local timer tick to process
  482. * the timer wheels. In that case, the expiry time can be before
  483. * jiffies, but since the high-resolution timer here is relative to
  484. * jiffies, the default expression when high-resolution timers are
  485. * not active,
  486. *
  487. * time_before(MAX_JIFFY_OFFSET + jiffies, expires)
  488. *
  489. * would falsely evaluate to true. If that is the case, just
  490. * return jiffies so that we can immediately fire the local timer
  491. */
  492. if (time_before(expires, jiffies))
  493. return jiffies;
  494. if (time_before(hr_expires, expires))
  495. return hr_expires;
  496. return expires;
  497. }
  498. #endif
  499. /******************************************************************/
  500. /*
  501. * The current time
  502. * wall_to_monotonic is what we need to add to xtime (or xtime corrected
  503. * for sub jiffie times) to get to monotonic time. Monotonic is pegged
  504. * at zero at system boot time, so wall_to_monotonic will be negative,
  505. * however, we will ALWAYS keep the tv_nsec part positive so we can use
  506. * the usual normalization.
  507. */
  508. struct timespec xtime __attribute__ ((aligned (16)));
  509. struct timespec wall_to_monotonic __attribute__ ((aligned (16)));
  510. EXPORT_SYMBOL(xtime);
  511. /* XXX - all of this timekeeping code should be later moved to time.c */
  512. #include <linux/clocksource.h>
  513. static struct clocksource *clock; /* pointer to current clocksource */
  514. #ifdef CONFIG_GENERIC_TIME
  515. /**
  516. * __get_nsec_offset - Returns nanoseconds since last call to periodic_hook
  517. *
  518. * private function, must hold xtime_lock lock when being
  519. * called. Returns the number of nanoseconds since the
  520. * last call to update_wall_time() (adjusted by NTP scaling)
  521. */
  522. static inline s64 __get_nsec_offset(void)
  523. {
  524. cycle_t cycle_now, cycle_delta;
  525. s64 ns_offset;
  526. /* read clocksource: */
  527. cycle_now = clocksource_read(clock);
  528. /* calculate the delta since the last update_wall_time: */
  529. cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
  530. /* convert to nanoseconds: */
  531. ns_offset = cyc2ns(clock, cycle_delta);
  532. return ns_offset;
  533. }
  534. /**
  535. * __get_realtime_clock_ts - Returns the time of day in a timespec
  536. * @ts: pointer to the timespec to be set
  537. *
  538. * Returns the time of day in a timespec. Used by
  539. * do_gettimeofday() and get_realtime_clock_ts().
  540. */
  541. static inline void __get_realtime_clock_ts(struct timespec *ts)
  542. {
  543. unsigned long seq;
  544. s64 nsecs;
  545. do {
  546. seq = read_seqbegin(&xtime_lock);
  547. *ts = xtime;
  548. nsecs = __get_nsec_offset();
  549. } while (read_seqretry(&xtime_lock, seq));
  550. timespec_add_ns(ts, nsecs);
  551. }
  552. /**
  553. * getnstimeofday - Returns the time of day in a timespec
  554. * @ts: pointer to the timespec to be set
  555. *
  556. * Returns the time of day in a timespec.
  557. */
  558. void getnstimeofday(struct timespec *ts)
  559. {
  560. __get_realtime_clock_ts(ts);
  561. }
  562. EXPORT_SYMBOL(getnstimeofday);
  563. /**
  564. * do_gettimeofday - Returns the time of day in a timeval
  565. * @tv: pointer to the timeval to be set
  566. *
  567. * NOTE: Users should be converted to using get_realtime_clock_ts()
  568. */
  569. void do_gettimeofday(struct timeval *tv)
  570. {
  571. struct timespec now;
  572. __get_realtime_clock_ts(&now);
  573. tv->tv_sec = now.tv_sec;
  574. tv->tv_usec = now.tv_nsec/1000;
  575. }
  576. EXPORT_SYMBOL(do_gettimeofday);
  577. /**
  578. * do_settimeofday - Sets the time of day
  579. * @tv: pointer to the timespec variable containing the new time
  580. *
  581. * Sets the time of day to the new time and update NTP and notify hrtimers
  582. */
  583. int do_settimeofday(struct timespec *tv)
  584. {
  585. unsigned long flags;
  586. time_t wtm_sec, sec = tv->tv_sec;
  587. long wtm_nsec, nsec = tv->tv_nsec;
  588. if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
  589. return -EINVAL;
  590. write_seqlock_irqsave(&xtime_lock, flags);
  591. nsec -= __get_nsec_offset();
  592. wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec);
  593. wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec);
  594. set_normalized_timespec(&xtime, sec, nsec);
  595. set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec);
  596. clock->error = 0;
  597. ntp_clear();
  598. write_sequnlock_irqrestore(&xtime_lock, flags);
  599. /* signal hrtimers about time change */
  600. clock_was_set();
  601. return 0;
  602. }
  603. EXPORT_SYMBOL(do_settimeofday);
  604. /**
  605. * change_clocksource - Swaps clocksources if a new one is available
  606. *
  607. * Accumulates current time interval and initializes new clocksource
  608. */
  609. static int change_clocksource(void)
  610. {
  611. struct clocksource *new;
  612. cycle_t now;
  613. u64 nsec;
  614. new = clocksource_get_next();
  615. if (clock != new) {
  616. now = clocksource_read(new);
  617. nsec = __get_nsec_offset();
  618. timespec_add_ns(&xtime, nsec);
  619. clock = new;
  620. clock->cycle_last = now;
  621. printk(KERN_INFO "Time: %s clocksource has been installed.\n",
  622. clock->name);
  623. return 1;
  624. } else if (clock->update_callback) {
  625. return clock->update_callback();
  626. }
  627. return 0;
  628. }
  629. #else
  630. #define change_clocksource() (0)
  631. #endif
  632. /**
  633. * timeofday_is_continuous - check to see if timekeeping is free running
  634. */
  635. int timekeeping_is_continuous(void)
  636. {
  637. unsigned long seq;
  638. int ret;
  639. do {
  640. seq = read_seqbegin(&xtime_lock);
  641. ret = clock->is_continuous;
  642. } while (read_seqretry(&xtime_lock, seq));
  643. return ret;
  644. }
  645. /*
  646. * timekeeping_init - Initializes the clocksource and common timekeeping values
  647. */
  648. void __init timekeeping_init(void)
  649. {
  650. unsigned long flags;
  651. write_seqlock_irqsave(&xtime_lock, flags);
  652. ntp_clear();
  653. clock = clocksource_get_next();
  654. clocksource_calculate_interval(clock, tick_nsec);
  655. clock->cycle_last = clocksource_read(clock);
  656. write_sequnlock_irqrestore(&xtime_lock, flags);
  657. }
  658. static int timekeeping_suspended;
  659. /**
  660. * timekeeping_resume - Resumes the generic timekeeping subsystem.
  661. * @dev: unused
  662. *
  663. * This is for the generic clocksource timekeeping.
  664. * xtime/wall_to_monotonic/jiffies/wall_jiffies/etc are
  665. * still managed by arch specific suspend/resume code.
  666. */
  667. static int timekeeping_resume(struct sys_device *dev)
  668. {
  669. unsigned long flags;
  670. write_seqlock_irqsave(&xtime_lock, flags);
  671. /* restart the last cycle value */
  672. clock->cycle_last = clocksource_read(clock);
  673. clock->error = 0;
  674. timekeeping_suspended = 0;
  675. write_sequnlock_irqrestore(&xtime_lock, flags);
  676. return 0;
  677. }
  678. static int timekeeping_suspend(struct sys_device *dev, pm_message_t state)
  679. {
  680. unsigned long flags;
  681. write_seqlock_irqsave(&xtime_lock, flags);
  682. timekeeping_suspended = 1;
  683. write_sequnlock_irqrestore(&xtime_lock, flags);
  684. return 0;
  685. }
  686. /* sysfs resume/suspend bits for timekeeping */
  687. static struct sysdev_class timekeeping_sysclass = {
  688. .resume = timekeeping_resume,
  689. .suspend = timekeeping_suspend,
  690. set_kset_name("timekeeping"),
  691. };
  692. static struct sys_device device_timer = {
  693. .id = 0,
  694. .cls = &timekeeping_sysclass,
  695. };
  696. static int __init timekeeping_init_device(void)
  697. {
  698. int error = sysdev_class_register(&timekeeping_sysclass);
  699. if (!error)
  700. error = sysdev_register(&device_timer);
  701. return error;
  702. }
  703. device_initcall(timekeeping_init_device);
  704. /*
  705. * If the error is already larger, we look ahead even further
  706. * to compensate for late or lost adjustments.
  707. */
  708. static __always_inline int clocksource_bigadjust(s64 error, s64 *interval, s64 *offset)
  709. {
  710. s64 tick_error, i;
  711. u32 look_ahead, adj;
  712. s32 error2, mult;
  713. /*
  714. * Use the current error value to determine how much to look ahead.
  715. * The larger the error the slower we adjust for it to avoid problems
  716. * with losing too many ticks, otherwise we would overadjust and
  717. * produce an even larger error. The smaller the adjustment the
  718. * faster we try to adjust for it, as lost ticks can do less harm
  719. * here. This is tuned so that an error of about 1 msec is adusted
  720. * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
  721. */
  722. error2 = clock->error >> (TICK_LENGTH_SHIFT + 22 - 2 * SHIFT_HZ);
  723. error2 = abs(error2);
  724. for (look_ahead = 0; error2 > 0; look_ahead++)
  725. error2 >>= 2;
  726. /*
  727. * Now calculate the error in (1 << look_ahead) ticks, but first
  728. * remove the single look ahead already included in the error.
  729. */
  730. tick_error = current_tick_length() >> (TICK_LENGTH_SHIFT - clock->shift + 1);
  731. tick_error -= clock->xtime_interval >> 1;
  732. error = ((error - tick_error) >> look_ahead) + tick_error;
  733. /* Finally calculate the adjustment shift value. */
  734. i = *interval;
  735. mult = 1;
  736. if (error < 0) {
  737. error = -error;
  738. *interval = -*interval;
  739. *offset = -*offset;
  740. mult = -1;
  741. }
  742. for (adj = 0; error > i; adj++)
  743. error >>= 1;
  744. *interval <<= adj;
  745. *offset <<= adj;
  746. return mult << adj;
  747. }
  748. /*
  749. * Adjust the multiplier to reduce the error value,
  750. * this is optimized for the most common adjustments of -1,0,1,
  751. * for other values we can do a bit more work.
  752. */
  753. static void clocksource_adjust(struct clocksource *clock, s64 offset)
  754. {
  755. s64 error, interval = clock->cycle_interval;
  756. int adj;
  757. error = clock->error >> (TICK_LENGTH_SHIFT - clock->shift - 1);
  758. if (error > interval) {
  759. error >>= 2;
  760. if (likely(error <= interval))
  761. adj = 1;
  762. else
  763. adj = clocksource_bigadjust(error, &interval, &offset);
  764. } else if (error < -interval) {
  765. error >>= 2;
  766. if (likely(error >= -interval)) {
  767. adj = -1;
  768. interval = -interval;
  769. offset = -offset;
  770. } else
  771. adj = clocksource_bigadjust(error, &interval, &offset);
  772. } else
  773. return;
  774. clock->mult += adj;
  775. clock->xtime_interval += interval;
  776. clock->xtime_nsec -= offset;
  777. clock->error -= (interval - offset) << (TICK_LENGTH_SHIFT - clock->shift);
  778. }
  779. /**
  780. * update_wall_time - Uses the current clocksource to increment the wall time
  781. *
  782. * Called from the timer interrupt, must hold a write on xtime_lock.
  783. */
  784. static void update_wall_time(void)
  785. {
  786. cycle_t offset;
  787. /* Make sure we're fully resumed: */
  788. if (unlikely(timekeeping_suspended))
  789. return;
  790. #ifdef CONFIG_GENERIC_TIME
  791. offset = (clocksource_read(clock) - clock->cycle_last) & clock->mask;
  792. #else
  793. offset = clock->cycle_interval;
  794. #endif
  795. clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift;
  796. /* normally this loop will run just once, however in the
  797. * case of lost or late ticks, it will accumulate correctly.
  798. */
  799. while (offset >= clock->cycle_interval) {
  800. /* accumulate one interval */
  801. clock->xtime_nsec += clock->xtime_interval;
  802. clock->cycle_last += clock->cycle_interval;
  803. offset -= clock->cycle_interval;
  804. if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) {
  805. clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift;
  806. xtime.tv_sec++;
  807. second_overflow();
  808. }
  809. /* interpolator bits */
  810. time_interpolator_update(clock->xtime_interval
  811. >> clock->shift);
  812. /* accumulate error between NTP and clock interval */
  813. clock->error += current_tick_length();
  814. clock->error -= clock->xtime_interval << (TICK_LENGTH_SHIFT - clock->shift);
  815. }
  816. /* correct the clock when NTP error is too big */
  817. clocksource_adjust(clock, offset);
  818. /* store full nanoseconds into xtime */
  819. xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift;
  820. clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
  821. /* check to see if there is a new clocksource to use */
  822. if (change_clocksource()) {
  823. clock->error = 0;
  824. clock->xtime_nsec = 0;
  825. clocksource_calculate_interval(clock, tick_nsec);
  826. }
  827. }
  828. /*
  829. * Called from the timer interrupt handler to charge one tick to the current
  830. * process. user_tick is 1 if the tick is user time, 0 for system.
  831. */
  832. void update_process_times(int user_tick)
  833. {
  834. struct task_struct *p = current;
  835. int cpu = smp_processor_id();
  836. /* Note: this timer irq context must be accounted for as well. */
  837. if (user_tick)
  838. account_user_time(p, jiffies_to_cputime(1));
  839. else
  840. account_system_time(p, HARDIRQ_OFFSET, jiffies_to_cputime(1));
  841. run_local_timers();
  842. if (rcu_pending(cpu))
  843. rcu_check_callbacks(cpu, user_tick);
  844. scheduler_tick();
  845. run_posix_cpu_timers(p);
  846. }
  847. /*
  848. * Nr of active tasks - counted in fixed-point numbers
  849. */
  850. static unsigned long count_active_tasks(void)
  851. {
  852. return nr_active() * FIXED_1;
  853. }
  854. /*
  855. * Hmm.. Changed this, as the GNU make sources (load.c) seems to
  856. * imply that avenrun[] is the standard name for this kind of thing.
  857. * Nothing else seems to be standardized: the fractional size etc
  858. * all seem to differ on different machines.
  859. *
  860. * Requires xtime_lock to access.
  861. */
  862. unsigned long avenrun[3];
  863. EXPORT_SYMBOL(avenrun);
  864. /*
  865. * calc_load - given tick count, update the avenrun load estimates.
  866. * This is called while holding a write_lock on xtime_lock.
  867. */
  868. static inline void calc_load(unsigned long ticks)
  869. {
  870. unsigned long active_tasks; /* fixed-point */
  871. static int count = LOAD_FREQ;
  872. active_tasks = count_active_tasks();
  873. for (count -= ticks; count < 0; count += LOAD_FREQ) {
  874. CALC_LOAD(avenrun[0], EXP_1, active_tasks);
  875. CALC_LOAD(avenrun[1], EXP_5, active_tasks);
  876. CALC_LOAD(avenrun[2], EXP_15, active_tasks);
  877. }
  878. }
  879. /* jiffies at the most recent update of wall time */
  880. unsigned long wall_jiffies = INITIAL_JIFFIES;
  881. /*
  882. * This read-write spinlock protects us from races in SMP while
  883. * playing with xtime and avenrun.
  884. */
  885. #ifndef ARCH_HAVE_XTIME_LOCK
  886. __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);
  887. EXPORT_SYMBOL(xtime_lock);
  888. #endif
  889. /*
  890. * This function runs timers and the timer-tq in bottom half context.
  891. */
  892. static void run_timer_softirq(struct softirq_action *h)
  893. {
  894. tvec_base_t *base = __get_cpu_var(tvec_bases);
  895. hrtimer_run_queues();
  896. if (time_after_eq(jiffies, base->timer_jiffies))
  897. __run_timers(base);
  898. }
  899. /*
  900. * Called by the local, per-CPU timer interrupt on SMP.
  901. */
  902. void run_local_timers(void)
  903. {
  904. raise_softirq(TIMER_SOFTIRQ);
  905. softlockup_tick();
  906. }
  907. /*
  908. * Called by the timer interrupt. xtime_lock must already be taken
  909. * by the timer IRQ!
  910. */
  911. static inline void update_times(unsigned long ticks)
  912. {
  913. wall_jiffies += ticks;
  914. update_wall_time();
  915. calc_load(ticks);
  916. }
  917. /*
  918. * The 64-bit jiffies value is not atomic - you MUST NOT read it
  919. * without sampling the sequence number in xtime_lock.
  920. * jiffies is defined in the linker script...
  921. */
  922. void do_timer(unsigned long ticks)
  923. {
  924. jiffies_64 += ticks;
  925. update_times(ticks);
  926. }
  927. #ifdef __ARCH_WANT_SYS_ALARM
  928. /*
  929. * For backwards compatibility? This can be done in libc so Alpha
  930. * and all newer ports shouldn't need it.
  931. */
  932. asmlinkage unsigned long sys_alarm(unsigned int seconds)
  933. {
  934. return alarm_setitimer(seconds);
  935. }
  936. #endif
  937. #ifndef __alpha__
  938. /*
  939. * The Alpha uses getxpid, getxuid, and getxgid instead. Maybe this
  940. * should be moved into arch/i386 instead?
  941. */
  942. /**
  943. * sys_getpid - return the thread group id of the current process
  944. *
  945. * Note, despite the name, this returns the tgid not the pid. The tgid and
  946. * the pid are identical unless CLONE_THREAD was specified on clone() in
  947. * which case the tgid is the same in all threads of the same group.
  948. *
  949. * This is SMP safe as current->tgid does not change.
  950. */
  951. asmlinkage long sys_getpid(void)
  952. {
  953. return current->tgid;
  954. }
  955. /*
  956. * Accessing ->real_parent is not SMP-safe, it could
  957. * change from under us. However, we can use a stale
  958. * value of ->real_parent under rcu_read_lock(), see
  959. * release_task()->call_rcu(delayed_put_task_struct).
  960. */
  961. asmlinkage long sys_getppid(void)
  962. {
  963. int pid;
  964. rcu_read_lock();
  965. pid = rcu_dereference(current->real_parent)->tgid;
  966. rcu_read_unlock();
  967. return pid;
  968. }
  969. asmlinkage long sys_getuid(void)
  970. {
  971. /* Only we change this so SMP safe */
  972. return current->uid;
  973. }
  974. asmlinkage long sys_geteuid(void)
  975. {
  976. /* Only we change this so SMP safe */
  977. return current->euid;
  978. }
  979. asmlinkage long sys_getgid(void)
  980. {
  981. /* Only we change this so SMP safe */
  982. return current->gid;
  983. }
  984. asmlinkage long sys_getegid(void)
  985. {
  986. /* Only we change this so SMP safe */
  987. return current->egid;
  988. }
  989. #endif
  990. static void process_timeout(unsigned long __data)
  991. {
  992. wake_up_process((struct task_struct *)__data);
  993. }
  994. /**
  995. * schedule_timeout - sleep until timeout
  996. * @timeout: timeout value in jiffies
  997. *
  998. * Make the current task sleep until @timeout jiffies have
  999. * elapsed. The routine will return immediately unless
  1000. * the current task state has been set (see set_current_state()).
  1001. *
  1002. * You can set the task state as follows -
  1003. *
  1004. * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
  1005. * pass before the routine returns. The routine will return 0
  1006. *
  1007. * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
  1008. * delivered to the current task. In this case the remaining time
  1009. * in jiffies will be returned, or 0 if the timer expired in time
  1010. *
  1011. * The current task state is guaranteed to be TASK_RUNNING when this
  1012. * routine returns.
  1013. *
  1014. * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
  1015. * the CPU away without a bound on the timeout. In this case the return
  1016. * value will be %MAX_SCHEDULE_TIMEOUT.
  1017. *
  1018. * In all cases the return value is guaranteed to be non-negative.
  1019. */
  1020. fastcall signed long __sched schedule_timeout(signed long timeout)
  1021. {
  1022. struct timer_list timer;
  1023. unsigned long expire;
  1024. switch (timeout)
  1025. {
  1026. case MAX_SCHEDULE_TIMEOUT:
  1027. /*
  1028. * These two special cases are useful to be comfortable
  1029. * in the caller. Nothing more. We could take
  1030. * MAX_SCHEDULE_TIMEOUT from one of the negative value
  1031. * but I' d like to return a valid offset (>=0) to allow
  1032. * the caller to do everything it want with the retval.
  1033. */
  1034. schedule();
  1035. goto out;
  1036. default:
  1037. /*
  1038. * Another bit of PARANOID. Note that the retval will be
  1039. * 0 since no piece of kernel is supposed to do a check
  1040. * for a negative retval of schedule_timeout() (since it
  1041. * should never happens anyway). You just have the printk()
  1042. * that will tell you if something is gone wrong and where.
  1043. */
  1044. if (timeout < 0)
  1045. {
  1046. printk(KERN_ERR "schedule_timeout: wrong timeout "
  1047. "value %lx from %p\n", timeout,
  1048. __builtin_return_address(0));
  1049. current->state = TASK_RUNNING;
  1050. goto out;
  1051. }
  1052. }
  1053. expire = timeout + jiffies;
  1054. setup_timer(&timer, process_timeout, (unsigned long)current);
  1055. __mod_timer(&timer, expire);
  1056. schedule();
  1057. del_singleshot_timer_sync(&timer);
  1058. timeout = expire - jiffies;
  1059. out:
  1060. return timeout < 0 ? 0 : timeout;
  1061. }
  1062. EXPORT_SYMBOL(schedule_timeout);
  1063. /*
  1064. * We can use __set_current_state() here because schedule_timeout() calls
  1065. * schedule() unconditionally.
  1066. */
  1067. signed long __sched schedule_timeout_interruptible(signed long timeout)
  1068. {
  1069. __set_current_state(TASK_INTERRUPTIBLE);
  1070. return schedule_timeout(timeout);
  1071. }
  1072. EXPORT_SYMBOL(schedule_timeout_interruptible);
  1073. signed long __sched schedule_timeout_uninterruptible(signed long timeout)
  1074. {
  1075. __set_current_state(TASK_UNINTERRUPTIBLE);
  1076. return schedule_timeout(timeout);
  1077. }
  1078. EXPORT_SYMBOL(schedule_timeout_uninterruptible);
  1079. /* Thread ID - the internal kernel "pid" */
  1080. asmlinkage long sys_gettid(void)
  1081. {
  1082. return current->pid;
  1083. }
  1084. /**
  1085. * sys_sysinfo - fill in sysinfo struct
  1086. * @info: pointer to buffer to fill
  1087. */
  1088. asmlinkage long sys_sysinfo(struct sysinfo __user *info)
  1089. {
  1090. struct sysinfo val;
  1091. unsigned long mem_total, sav_total;
  1092. unsigned int mem_unit, bitcount;
  1093. unsigned long seq;
  1094. memset((char *)&val, 0, sizeof(struct sysinfo));
  1095. do {
  1096. struct timespec tp;
  1097. seq = read_seqbegin(&xtime_lock);
  1098. /*
  1099. * This is annoying. The below is the same thing
  1100. * posix_get_clock_monotonic() does, but it wants to
  1101. * take the lock which we want to cover the loads stuff
  1102. * too.
  1103. */
  1104. getnstimeofday(&tp);
  1105. tp.tv_sec += wall_to_monotonic.tv_sec;
  1106. tp.tv_nsec += wall_to_monotonic.tv_nsec;
  1107. if (tp.tv_nsec - NSEC_PER_SEC >= 0) {
  1108. tp.tv_nsec = tp.tv_nsec - NSEC_PER_SEC;
  1109. tp.tv_sec++;
  1110. }
  1111. val.uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0);
  1112. val.loads[0] = avenrun[0] << (SI_LOAD_SHIFT - FSHIFT);
  1113. val.loads[1] = avenrun[1] << (SI_LOAD_SHIFT - FSHIFT);
  1114. val.loads[2] = avenrun[2] << (SI_LOAD_SHIFT - FSHIFT);
  1115. val.procs = nr_threads;
  1116. } while (read_seqretry(&xtime_lock, seq));
  1117. si_meminfo(&val);
  1118. si_swapinfo(&val);
  1119. /*
  1120. * If the sum of all the available memory (i.e. ram + swap)
  1121. * is less than can be stored in a 32 bit unsigned long then
  1122. * we can be binary compatible with 2.2.x kernels. If not,
  1123. * well, in that case 2.2.x was broken anyways...
  1124. *
  1125. * -Erik Andersen <andersee@debian.org>
  1126. */
  1127. mem_total = val.totalram + val.totalswap;
  1128. if (mem_total < val.totalram || mem_total < val.totalswap)
  1129. goto out;
  1130. bitcount = 0;
  1131. mem_unit = val.mem_unit;
  1132. while (mem_unit > 1) {
  1133. bitcount++;
  1134. mem_unit >>= 1;
  1135. sav_total = mem_total;
  1136. mem_total <<= 1;
  1137. if (mem_total < sav_total)
  1138. goto out;
  1139. }
  1140. /*
  1141. * If mem_total did not overflow, multiply all memory values by
  1142. * val.mem_unit and set it to 1. This leaves things compatible
  1143. * with 2.2.x, and also retains compatibility with earlier 2.4.x
  1144. * kernels...
  1145. */
  1146. val.mem_unit = 1;
  1147. val.totalram <<= bitcount;
  1148. val.freeram <<= bitcount;
  1149. val.sharedram <<= bitcount;
  1150. val.bufferram <<= bitcount;
  1151. val.totalswap <<= bitcount;
  1152. val.freeswap <<= bitcount;
  1153. val.totalhigh <<= bitcount;
  1154. val.freehigh <<= bitcount;
  1155. out:
  1156. if (copy_to_user(info, &val, sizeof(struct sysinfo)))
  1157. return -EFAULT;
  1158. return 0;
  1159. }
  1160. /*
  1161. * lockdep: we want to track each per-CPU base as a separate lock-class,
  1162. * but timer-bases are kmalloc()-ed, so we need to attach separate
  1163. * keys to them:
  1164. */
  1165. static struct lock_class_key base_lock_keys[NR_CPUS];
  1166. static int __devinit init_timers_cpu(int cpu)
  1167. {
  1168. int j;
  1169. tvec_base_t *base;
  1170. static char __devinitdata tvec_base_done[NR_CPUS];
  1171. if (!tvec_base_done[cpu]) {
  1172. static char boot_done;
  1173. if (boot_done) {
  1174. /*
  1175. * The APs use this path later in boot
  1176. */
  1177. base = kmalloc_node(sizeof(*base), GFP_KERNEL,
  1178. cpu_to_node(cpu));
  1179. if (!base)
  1180. return -ENOMEM;
  1181. memset(base, 0, sizeof(*base));
  1182. per_cpu(tvec_bases, cpu) = base;
  1183. } else {
  1184. /*
  1185. * This is for the boot CPU - we use compile-time
  1186. * static initialisation because per-cpu memory isn't
  1187. * ready yet and because the memory allocators are not
  1188. * initialised either.
  1189. */
  1190. boot_done = 1;
  1191. base = &boot_tvec_bases;
  1192. }
  1193. tvec_base_done[cpu] = 1;
  1194. } else {
  1195. base = per_cpu(tvec_bases, cpu);
  1196. }
  1197. spin_lock_init(&base->lock);
  1198. lockdep_set_class(&base->lock, base_lock_keys + cpu);
  1199. for (j = 0; j < TVN_SIZE; j++) {
  1200. INIT_LIST_HEAD(base->tv5.vec + j);
  1201. INIT_LIST_HEAD(base->tv4.vec + j);
  1202. INIT_LIST_HEAD(base->tv3.vec + j);
  1203. INIT_LIST_HEAD(base->tv2.vec + j);
  1204. }
  1205. for (j = 0; j < TVR_SIZE; j++)
  1206. INIT_LIST_HEAD(base->tv1.vec + j);
  1207. base->timer_jiffies = jiffies;
  1208. return 0;
  1209. }
  1210. #ifdef CONFIG_HOTPLUG_CPU
  1211. static void migrate_timer_list(tvec_base_t *new_base, struct list_head *head)
  1212. {
  1213. struct timer_list *timer;
  1214. while (!list_empty(head)) {
  1215. timer = list_entry(head->next, struct timer_list, entry);
  1216. detach_timer(timer, 0);
  1217. timer->base = new_base;
  1218. internal_add_timer(new_base, timer);
  1219. }
  1220. }
  1221. static void __devinit migrate_timers(int cpu)
  1222. {
  1223. tvec_base_t *old_base;
  1224. tvec_base_t *new_base;
  1225. int i;
  1226. BUG_ON(cpu_online(cpu));
  1227. old_base = per_cpu(tvec_bases, cpu);
  1228. new_base = get_cpu_var(tvec_bases);
  1229. local_irq_disable();
  1230. spin_lock(&new_base->lock);
  1231. spin_lock(&old_base->lock);
  1232. BUG_ON(old_base->running_timer);
  1233. for (i = 0; i < TVR_SIZE; i++)
  1234. migrate_timer_list(new_base, old_base->tv1.vec + i);
  1235. for (i = 0; i < TVN_SIZE; i++) {
  1236. migrate_timer_list(new_base, old_base->tv2.vec + i);
  1237. migrate_timer_list(new_base, old_base->tv3.vec + i);
  1238. migrate_timer_list(new_base, old_base->tv4.vec + i);
  1239. migrate_timer_list(new_base, old_base->tv5.vec + i);
  1240. }
  1241. spin_unlock(&old_base->lock);
  1242. spin_unlock(&new_base->lock);
  1243. local_irq_enable();
  1244. put_cpu_var(tvec_bases);
  1245. }
  1246. #endif /* CONFIG_HOTPLUG_CPU */
  1247. static int __cpuinit timer_cpu_notify(struct notifier_block *self,
  1248. unsigned long action, void *hcpu)
  1249. {
  1250. long cpu = (long)hcpu;
  1251. switch(action) {
  1252. case CPU_UP_PREPARE:
  1253. if (init_timers_cpu(cpu) < 0)
  1254. return NOTIFY_BAD;
  1255. break;
  1256. #ifdef CONFIG_HOTPLUG_CPU
  1257. case CPU_DEAD:
  1258. migrate_timers(cpu);
  1259. break;
  1260. #endif
  1261. default:
  1262. break;
  1263. }
  1264. return NOTIFY_OK;
  1265. }
  1266. static struct notifier_block __cpuinitdata timers_nb = {
  1267. .notifier_call = timer_cpu_notify,
  1268. };
  1269. void __init init_timers(void)
  1270. {
  1271. int err = timer_cpu_notify(&timers_nb, (unsigned long)CPU_UP_PREPARE,
  1272. (void *)(long)smp_processor_id());
  1273. BUG_ON(err == NOTIFY_BAD);
  1274. register_cpu_notifier(&timers_nb);
  1275. open_softirq(TIMER_SOFTIRQ, run_timer_softirq, NULL);
  1276. }
  1277. #ifdef CONFIG_TIME_INTERPOLATION
  1278. struct time_interpolator *time_interpolator __read_mostly;
  1279. static struct time_interpolator *time_interpolator_list __read_mostly;
  1280. static DEFINE_SPINLOCK(time_interpolator_lock);
  1281. static inline u64 time_interpolator_get_cycles(unsigned int src)
  1282. {
  1283. unsigned long (*x)(void);
  1284. switch (src)
  1285. {
  1286. case TIME_SOURCE_FUNCTION:
  1287. x = time_interpolator->addr;
  1288. return x();
  1289. case TIME_SOURCE_MMIO64 :
  1290. return readq_relaxed((void __iomem *)time_interpolator->addr);
  1291. case TIME_SOURCE_MMIO32 :
  1292. return readl_relaxed((void __iomem *)time_interpolator->addr);
  1293. default: return get_cycles();
  1294. }
  1295. }
  1296. static inline u64 time_interpolator_get_counter(int writelock)
  1297. {
  1298. unsigned int src = time_interpolator->source;
  1299. if (time_interpolator->jitter)
  1300. {
  1301. u64 lcycle;
  1302. u64 now;
  1303. do {
  1304. lcycle = time_interpolator->last_cycle;
  1305. now = time_interpolator_get_cycles(src);
  1306. if (lcycle && time_after(lcycle, now))
  1307. return lcycle;
  1308. /* When holding the xtime write lock, there's no need
  1309. * to add the overhead of the cmpxchg. Readers are
  1310. * force to retry until the write lock is released.
  1311. */
  1312. if (writelock) {
  1313. time_interpolator->last_cycle = now;
  1314. return now;
  1315. }
  1316. /* Keep track of the last timer value returned. The use of cmpxchg here
  1317. * will cause contention in an SMP environment.
  1318. */
  1319. } while (unlikely(cmpxchg(&time_interpolator->last_cycle, lcycle, now) != lcycle));
  1320. return now;
  1321. }
  1322. else
  1323. return time_interpolator_get_cycles(src);
  1324. }
  1325. void time_interpolator_reset(void)
  1326. {
  1327. time_interpolator->offset = 0;
  1328. time_interpolator->last_counter = time_interpolator_get_counter(1);
  1329. }
  1330. #define GET_TI_NSECS(count,i) (((((count) - i->last_counter) & (i)->mask) * (i)->nsec_per_cyc) >> (i)->shift)
  1331. unsigned long time_interpolator_get_offset(void)
  1332. {
  1333. /* If we do not have a time interpolator set up then just return zero */
  1334. if (!time_interpolator)
  1335. return 0;
  1336. return time_interpolator->offset +
  1337. GET_TI_NSECS(time_interpolator_get_counter(0), time_interpolator);
  1338. }
  1339. #define INTERPOLATOR_ADJUST 65536
  1340. #define INTERPOLATOR_MAX_SKIP 10*INTERPOLATOR_ADJUST
  1341. void time_interpolator_update(long delta_nsec)
  1342. {
  1343. u64 counter;
  1344. unsigned long offset;
  1345. /* If there is no time interpolator set up then do nothing */
  1346. if (!time_interpolator)
  1347. return;
  1348. /*
  1349. * The interpolator compensates for late ticks by accumulating the late
  1350. * time in time_interpolator->offset. A tick earlier than expected will
  1351. * lead to a reset of the offset and a corresponding jump of the clock
  1352. * forward. Again this only works if the interpolator clock is running
  1353. * slightly slower than the regular clock and the tuning logic insures
  1354. * that.
  1355. */
  1356. counter = time_interpolator_get_counter(1);
  1357. offset = time_interpolator->offset +
  1358. GET_TI_NSECS(counter, time_interpolator);
  1359. if (delta_nsec < 0 || (unsigned long) delta_nsec < offset)
  1360. time_interpolator->offset = offset - delta_nsec;
  1361. else {
  1362. time_interpolator->skips++;
  1363. time_interpolator->ns_skipped += delta_nsec - offset;
  1364. time_interpolator->offset = 0;
  1365. }
  1366. time_interpolator->last_counter = counter;
  1367. /* Tuning logic for time interpolator invoked every minute or so.
  1368. * Decrease interpolator clock speed if no skips occurred and an offset is carried.
  1369. * Increase interpolator clock speed if we skip too much time.
  1370. */
  1371. if (jiffies % INTERPOLATOR_ADJUST == 0)
  1372. {
  1373. if (time_interpolator->skips == 0 && time_interpolator->offset > tick_nsec)
  1374. time_interpolator->nsec_per_cyc--;
  1375. if (time_interpolator->ns_skipped > INTERPOLATOR_MAX_SKIP && time_interpolator->offset == 0)
  1376. time_interpolator->nsec_per_cyc++;
  1377. time_interpolator->skips = 0;
  1378. time_interpolator->ns_skipped = 0;
  1379. }
  1380. }
  1381. static inline int
  1382. is_better_time_interpolator(struct time_interpolator *new)
  1383. {
  1384. if (!time_interpolator)
  1385. return 1;
  1386. return new->frequency > 2*time_interpolator->frequency ||
  1387. (unsigned long)new->drift < (unsigned long)time_interpolator->drift;
  1388. }
  1389. void
  1390. register_time_interpolator(struct time_interpolator *ti)
  1391. {
  1392. unsigned long flags;
  1393. /* Sanity check */
  1394. BUG_ON(ti->frequency == 0 || ti->mask == 0);
  1395. ti->nsec_per_cyc = ((u64)NSEC_PER_SEC << ti->shift) / ti->frequency;
  1396. spin_lock(&time_interpolator_lock);
  1397. write_seqlock_irqsave(&xtime_lock, flags);
  1398. if (is_better_time_interpolator(ti)) {
  1399. time_interpolator = ti;
  1400. time_interpolator_reset();
  1401. }
  1402. write_sequnlock_irqrestore(&xtime_lock, flags);
  1403. ti->next = time_interpolator_list;
  1404. time_interpolator_list = ti;
  1405. spin_unlock(&time_interpolator_lock);
  1406. }
  1407. void
  1408. unregister_time_interpolator(struct time_interpolator *ti)
  1409. {
  1410. struct time_interpolator *curr, **prev;
  1411. unsigned long flags;
  1412. spin_lock(&time_interpolator_lock);
  1413. prev = &time_interpolator_list;
  1414. for (curr = *prev; curr; curr = curr->next) {
  1415. if (curr == ti) {
  1416. *prev = curr->next;
  1417. break;
  1418. }
  1419. prev = &curr->next;
  1420. }
  1421. write_seqlock_irqsave(&xtime_lock, flags);
  1422. if (ti == time_interpolator) {
  1423. /* we lost the best time-interpolator: */
  1424. time_interpolator = NULL;
  1425. /* find the next-best interpolator */
  1426. for (curr = time_interpolator_list; curr; curr = curr->next)
  1427. if (is_better_time_interpolator(curr))
  1428. time_interpolator = curr;
  1429. time_interpolator_reset();
  1430. }
  1431. write_sequnlock_irqrestore(&xtime_lock, flags);
  1432. spin_unlock(&time_interpolator_lock);
  1433. }
  1434. #endif /* CONFIG_TIME_INTERPOLATION */
  1435. /**
  1436. * msleep - sleep safely even with waitqueue interruptions
  1437. * @msecs: Time in milliseconds to sleep for
  1438. */
  1439. void msleep(unsigned int msecs)
  1440. {
  1441. unsigned long timeout = msecs_to_jiffies(msecs) + 1;
  1442. while (timeout)
  1443. timeout = schedule_timeout_uninterruptible(timeout);
  1444. }
  1445. EXPORT_SYMBOL(msleep);
  1446. /**
  1447. * msleep_interruptible - sleep waiting for signals
  1448. * @msecs: Time in milliseconds to sleep for
  1449. */
  1450. unsigned long msleep_interruptible(unsigned int msecs)
  1451. {
  1452. unsigned long timeout = msecs_to_jiffies(msecs) + 1;
  1453. while (timeout && !signal_pending(current))
  1454. timeout = schedule_timeout_interruptible(timeout);
  1455. return jiffies_to_msecs(timeout);
  1456. }
  1457. EXPORT_SYMBOL(msleep_interruptible);