time.c 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203
  1. /*
  2. * Common time routines among all ppc machines.
  3. *
  4. * Written by Cort Dougan (cort@cs.nmt.edu) to merge
  5. * Paul Mackerras' version and mine for PReP and Pmac.
  6. * MPC8xx/MBX changes by Dan Malek (dmalek@jlc.net).
  7. * Converted for 64-bit by Mike Corrigan (mikejc@us.ibm.com)
  8. *
  9. * First round of bugfixes by Gabriel Paubert (paubert@iram.es)
  10. * to make clock more stable (2.4.0-test5). The only thing
  11. * that this code assumes is that the timebases have been synchronized
  12. * by firmware on SMP and are never stopped (never do sleep
  13. * on SMP then, nap and doze are OK).
  14. *
  15. * Speeded up do_gettimeofday by getting rid of references to
  16. * xtime (which required locks for consistency). (mikejc@us.ibm.com)
  17. *
  18. * TODO (not necessarily in this file):
  19. * - improve precision and reproducibility of timebase frequency
  20. * measurement at boot time. (for iSeries, we calibrate the timebase
  21. * against the Titan chip's clock.)
  22. * - for astronomical applications: add a new function to get
  23. * non ambiguous timestamps even around leap seconds. This needs
  24. * a new timestamp format and a good name.
  25. *
  26. * 1997-09-10 Updated NTP code according to technical memorandum Jan '96
  27. * "A Kernel Model for Precision Timekeeping" by Dave Mills
  28. *
  29. * This program is free software; you can redistribute it and/or
  30. * modify it under the terms of the GNU General Public License
  31. * as published by the Free Software Foundation; either version
  32. * 2 of the License, or (at your option) any later version.
  33. */
  34. #include <linux/errno.h>
  35. #include <linux/module.h>
  36. #include <linux/sched.h>
  37. #include <linux/kernel.h>
  38. #include <linux/param.h>
  39. #include <linux/string.h>
  40. #include <linux/mm.h>
  41. #include <linux/interrupt.h>
  42. #include <linux/timex.h>
  43. #include <linux/kernel_stat.h>
  44. #include <linux/time.h>
  45. #include <linux/init.h>
  46. #include <linux/profile.h>
  47. #include <linux/cpu.h>
  48. #include <linux/security.h>
  49. #include <linux/percpu.h>
  50. #include <linux/rtc.h>
  51. #include <linux/jiffies.h>
  52. #include <linux/posix-timers.h>
  53. #include <linux/irq.h>
  54. #include <asm/io.h>
  55. #include <asm/processor.h>
  56. #include <asm/nvram.h>
  57. #include <asm/cache.h>
  58. #include <asm/machdep.h>
  59. #include <asm/uaccess.h>
  60. #include <asm/time.h>
  61. #include <asm/prom.h>
  62. #include <asm/irq.h>
  63. #include <asm/div64.h>
  64. #include <asm/smp.h>
  65. #include <asm/vdso_datapage.h>
  66. #ifdef CONFIG_PPC64
  67. #include <asm/firmware.h>
  68. #endif
  69. #ifdef CONFIG_PPC_ISERIES
  70. #include <asm/iseries/it_lp_queue.h>
  71. #include <asm/iseries/hv_call_xm.h>
  72. #endif
  73. #include <asm/smp.h>
  74. /* keep track of when we need to update the rtc */
  75. time_t last_rtc_update;
  76. #ifdef CONFIG_PPC_ISERIES
  77. unsigned long iSeries_recal_titan = 0;
  78. unsigned long iSeries_recal_tb = 0;
  79. static unsigned long first_settimeofday = 1;
  80. #endif
  81. /* The decrementer counts down by 128 every 128ns on a 601. */
  82. #define DECREMENTER_COUNT_601 (1000000000 / HZ)
  83. #define XSEC_PER_SEC (1024*1024)
  84. #ifdef CONFIG_PPC64
  85. #define SCALE_XSEC(xsec, max) (((xsec) * max) / XSEC_PER_SEC)
  86. #else
  87. /* compute ((xsec << 12) * max) >> 32 */
  88. #define SCALE_XSEC(xsec, max) mulhwu((xsec) << 12, max)
  89. #endif
  90. unsigned long tb_ticks_per_jiffy;
  91. unsigned long tb_ticks_per_usec = 100; /* sane default */
  92. EXPORT_SYMBOL(tb_ticks_per_usec);
  93. unsigned long tb_ticks_per_sec;
  94. EXPORT_SYMBOL(tb_ticks_per_sec); /* for cputime_t conversions */
  95. u64 tb_to_xs;
  96. unsigned tb_to_us;
  97. #define TICKLEN_SCALE TICK_LENGTH_SHIFT
  98. u64 last_tick_len; /* units are ns / 2^TICKLEN_SCALE */
  99. u64 ticklen_to_xs; /* 0.64 fraction */
  100. /* If last_tick_len corresponds to about 1/HZ seconds, then
  101. last_tick_len << TICKLEN_SHIFT will be about 2^63. */
  102. #define TICKLEN_SHIFT (63 - 30 - TICKLEN_SCALE + SHIFT_HZ)
  103. DEFINE_SPINLOCK(rtc_lock);
  104. EXPORT_SYMBOL_GPL(rtc_lock);
  105. u64 tb_to_ns_scale;
  106. unsigned tb_to_ns_shift;
  107. struct gettimeofday_struct do_gtod;
  108. extern struct timezone sys_tz;
  109. static long timezone_offset;
  110. unsigned long ppc_proc_freq;
  111. unsigned long ppc_tb_freq;
  112. static u64 tb_last_jiffy __cacheline_aligned_in_smp;
  113. static DEFINE_PER_CPU(u64, last_jiffy);
  114. #ifdef CONFIG_VIRT_CPU_ACCOUNTING
  115. /*
  116. * Factors for converting from cputime_t (timebase ticks) to
  117. * jiffies, milliseconds, seconds, and clock_t (1/USER_HZ seconds).
  118. * These are all stored as 0.64 fixed-point binary fractions.
  119. */
  120. u64 __cputime_jiffies_factor;
  121. EXPORT_SYMBOL(__cputime_jiffies_factor);
  122. u64 __cputime_msec_factor;
  123. EXPORT_SYMBOL(__cputime_msec_factor);
  124. u64 __cputime_sec_factor;
  125. EXPORT_SYMBOL(__cputime_sec_factor);
  126. u64 __cputime_clockt_factor;
  127. EXPORT_SYMBOL(__cputime_clockt_factor);
  128. static void calc_cputime_factors(void)
  129. {
  130. struct div_result res;
  131. div128_by_32(HZ, 0, tb_ticks_per_sec, &res);
  132. __cputime_jiffies_factor = res.result_low;
  133. div128_by_32(1000, 0, tb_ticks_per_sec, &res);
  134. __cputime_msec_factor = res.result_low;
  135. div128_by_32(1, 0, tb_ticks_per_sec, &res);
  136. __cputime_sec_factor = res.result_low;
  137. div128_by_32(USER_HZ, 0, tb_ticks_per_sec, &res);
  138. __cputime_clockt_factor = res.result_low;
  139. }
  140. /*
  141. * Read the PURR on systems that have it, otherwise the timebase.
  142. */
  143. static u64 read_purr(void)
  144. {
  145. if (cpu_has_feature(CPU_FTR_PURR))
  146. return mfspr(SPRN_PURR);
  147. return mftb();
  148. }
  149. /*
  150. * Account time for a transition between system, hard irq
  151. * or soft irq state.
  152. */
  153. void account_system_vtime(struct task_struct *tsk)
  154. {
  155. u64 now, delta;
  156. unsigned long flags;
  157. local_irq_save(flags);
  158. now = read_purr();
  159. delta = now - get_paca()->startpurr;
  160. get_paca()->startpurr = now;
  161. if (!in_interrupt()) {
  162. delta += get_paca()->system_time;
  163. get_paca()->system_time = 0;
  164. }
  165. account_system_time(tsk, 0, delta);
  166. local_irq_restore(flags);
  167. }
  168. /*
  169. * Transfer the user and system times accumulated in the paca
  170. * by the exception entry and exit code to the generic process
  171. * user and system time records.
  172. * Must be called with interrupts disabled.
  173. */
  174. void account_process_vtime(struct task_struct *tsk)
  175. {
  176. cputime_t utime;
  177. utime = get_paca()->user_time;
  178. get_paca()->user_time = 0;
  179. account_user_time(tsk, utime);
  180. }
  181. static void account_process_time(struct pt_regs *regs)
  182. {
  183. int cpu = smp_processor_id();
  184. account_process_vtime(current);
  185. run_local_timers();
  186. if (rcu_pending(cpu))
  187. rcu_check_callbacks(cpu, user_mode(regs));
  188. scheduler_tick();
  189. run_posix_cpu_timers(current);
  190. }
  191. #ifdef CONFIG_PPC_SPLPAR
  192. /*
  193. * Stuff for accounting stolen time.
  194. */
  195. struct cpu_purr_data {
  196. int initialized; /* thread is running */
  197. u64 tb; /* last TB value read */
  198. u64 purr; /* last PURR value read */
  199. spinlock_t lock;
  200. };
  201. static DEFINE_PER_CPU(struct cpu_purr_data, cpu_purr_data);
  202. static void snapshot_tb_and_purr(void *data)
  203. {
  204. struct cpu_purr_data *p = &__get_cpu_var(cpu_purr_data);
  205. p->tb = mftb();
  206. p->purr = mfspr(SPRN_PURR);
  207. wmb();
  208. p->initialized = 1;
  209. }
  210. /*
  211. * Called during boot when all cpus have come up.
  212. */
  213. void snapshot_timebases(void)
  214. {
  215. int cpu;
  216. if (!cpu_has_feature(CPU_FTR_PURR))
  217. return;
  218. for_each_possible_cpu(cpu)
  219. spin_lock_init(&per_cpu(cpu_purr_data, cpu).lock);
  220. on_each_cpu(snapshot_tb_and_purr, NULL, 0, 1);
  221. }
  222. void calculate_steal_time(void)
  223. {
  224. u64 tb, purr;
  225. s64 stolen;
  226. struct cpu_purr_data *pme;
  227. if (!cpu_has_feature(CPU_FTR_PURR))
  228. return;
  229. pme = &per_cpu(cpu_purr_data, smp_processor_id());
  230. if (!pme->initialized)
  231. return; /* this can happen in early boot */
  232. spin_lock(&pme->lock);
  233. tb = mftb();
  234. purr = mfspr(SPRN_PURR);
  235. stolen = (tb - pme->tb) - (purr - pme->purr);
  236. if (stolen > 0)
  237. account_steal_time(current, stolen);
  238. pme->tb = tb;
  239. pme->purr = purr;
  240. spin_unlock(&pme->lock);
  241. }
  242. /*
  243. * Must be called before the cpu is added to the online map when
  244. * a cpu is being brought up at runtime.
  245. */
  246. static void snapshot_purr(void)
  247. {
  248. struct cpu_purr_data *pme;
  249. unsigned long flags;
  250. if (!cpu_has_feature(CPU_FTR_PURR))
  251. return;
  252. pme = &per_cpu(cpu_purr_data, smp_processor_id());
  253. spin_lock_irqsave(&pme->lock, flags);
  254. pme->tb = mftb();
  255. pme->purr = mfspr(SPRN_PURR);
  256. pme->initialized = 1;
  257. spin_unlock_irqrestore(&pme->lock, flags);
  258. }
  259. #endif /* CONFIG_PPC_SPLPAR */
  260. #else /* ! CONFIG_VIRT_CPU_ACCOUNTING */
  261. #define calc_cputime_factors()
  262. #define account_process_time(regs) update_process_times(user_mode(regs))
  263. #define calculate_steal_time() do { } while (0)
  264. #endif
  265. #if !(defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(CONFIG_PPC_SPLPAR))
  266. #define snapshot_purr() do { } while (0)
  267. #endif
  268. /*
  269. * Called when a cpu comes up after the system has finished booting,
  270. * i.e. as a result of a hotplug cpu action.
  271. */
  272. void snapshot_timebase(void)
  273. {
  274. __get_cpu_var(last_jiffy) = get_tb();
  275. snapshot_purr();
  276. }
  277. void __delay(unsigned long loops)
  278. {
  279. unsigned long start;
  280. int diff;
  281. if (__USE_RTC()) {
  282. start = get_rtcl();
  283. do {
  284. /* the RTCL register wraps at 1000000000 */
  285. diff = get_rtcl() - start;
  286. if (diff < 0)
  287. diff += 1000000000;
  288. } while (diff < loops);
  289. } else {
  290. start = get_tbl();
  291. while (get_tbl() - start < loops)
  292. HMT_low();
  293. HMT_medium();
  294. }
  295. }
  296. EXPORT_SYMBOL(__delay);
  297. void udelay(unsigned long usecs)
  298. {
  299. __delay(tb_ticks_per_usec * usecs);
  300. }
  301. EXPORT_SYMBOL(udelay);
  302. static __inline__ void timer_check_rtc(void)
  303. {
  304. /*
  305. * update the rtc when needed, this should be performed on the
  306. * right fraction of a second. Half or full second ?
  307. * Full second works on mk48t59 clocks, others need testing.
  308. * Note that this update is basically only used through
  309. * the adjtimex system calls. Setting the HW clock in
  310. * any other way is a /dev/rtc and userland business.
  311. * This is still wrong by -0.5/+1.5 jiffies because of the
  312. * timer interrupt resolution and possible delay, but here we
  313. * hit a quantization limit which can only be solved by higher
  314. * resolution timers and decoupling time management from timer
  315. * interrupts. This is also wrong on the clocks
  316. * which require being written at the half second boundary.
  317. * We should have an rtc call that only sets the minutes and
  318. * seconds like on Intel to avoid problems with non UTC clocks.
  319. */
  320. if (ppc_md.set_rtc_time && ntp_synced() &&
  321. xtime.tv_sec - last_rtc_update >= 659 &&
  322. abs((xtime.tv_nsec/1000) - (1000000-1000000/HZ)) < 500000/HZ) {
  323. struct rtc_time tm;
  324. to_tm(xtime.tv_sec + 1 + timezone_offset, &tm);
  325. tm.tm_year -= 1900;
  326. tm.tm_mon -= 1;
  327. if (ppc_md.set_rtc_time(&tm) == 0)
  328. last_rtc_update = xtime.tv_sec + 1;
  329. else
  330. /* Try again one minute later */
  331. last_rtc_update += 60;
  332. }
  333. }
  334. /*
  335. * This version of gettimeofday has microsecond resolution.
  336. */
  337. static inline void __do_gettimeofday(struct timeval *tv)
  338. {
  339. unsigned long sec, usec;
  340. u64 tb_ticks, xsec;
  341. struct gettimeofday_vars *temp_varp;
  342. u64 temp_tb_to_xs, temp_stamp_xsec;
  343. /*
  344. * These calculations are faster (gets rid of divides)
  345. * if done in units of 1/2^20 rather than microseconds.
  346. * The conversion to microseconds at the end is done
  347. * without a divide (and in fact, without a multiply)
  348. */
  349. temp_varp = do_gtod.varp;
  350. /* Sampling the time base must be done after loading
  351. * do_gtod.varp in order to avoid racing with update_gtod.
  352. */
  353. data_barrier(temp_varp);
  354. tb_ticks = get_tb() - temp_varp->tb_orig_stamp;
  355. temp_tb_to_xs = temp_varp->tb_to_xs;
  356. temp_stamp_xsec = temp_varp->stamp_xsec;
  357. xsec = temp_stamp_xsec + mulhdu(tb_ticks, temp_tb_to_xs);
  358. sec = xsec / XSEC_PER_SEC;
  359. usec = (unsigned long)xsec & (XSEC_PER_SEC - 1);
  360. usec = SCALE_XSEC(usec, 1000000);
  361. tv->tv_sec = sec;
  362. tv->tv_usec = usec;
  363. }
  364. void do_gettimeofday(struct timeval *tv)
  365. {
  366. if (__USE_RTC()) {
  367. /* do this the old way */
  368. unsigned long flags, seq;
  369. unsigned int sec, nsec, usec;
  370. do {
  371. seq = read_seqbegin_irqsave(&xtime_lock, flags);
  372. sec = xtime.tv_sec;
  373. nsec = xtime.tv_nsec + tb_ticks_since(tb_last_jiffy);
  374. } while (read_seqretry_irqrestore(&xtime_lock, seq, flags));
  375. usec = nsec / 1000;
  376. while (usec >= 1000000) {
  377. usec -= 1000000;
  378. ++sec;
  379. }
  380. tv->tv_sec = sec;
  381. tv->tv_usec = usec;
  382. return;
  383. }
  384. __do_gettimeofday(tv);
  385. }
  386. EXPORT_SYMBOL(do_gettimeofday);
  387. /*
  388. * There are two copies of tb_to_xs and stamp_xsec so that no
  389. * lock is needed to access and use these values in
  390. * do_gettimeofday. We alternate the copies and as long as a
  391. * reasonable time elapses between changes, there will never
  392. * be inconsistent values. ntpd has a minimum of one minute
  393. * between updates.
  394. */
  395. static inline void update_gtod(u64 new_tb_stamp, u64 new_stamp_xsec,
  396. u64 new_tb_to_xs)
  397. {
  398. unsigned temp_idx;
  399. struct gettimeofday_vars *temp_varp;
  400. temp_idx = (do_gtod.var_idx == 0);
  401. temp_varp = &do_gtod.vars[temp_idx];
  402. temp_varp->tb_to_xs = new_tb_to_xs;
  403. temp_varp->tb_orig_stamp = new_tb_stamp;
  404. temp_varp->stamp_xsec = new_stamp_xsec;
  405. smp_mb();
  406. do_gtod.varp = temp_varp;
  407. do_gtod.var_idx = temp_idx;
  408. /*
  409. * tb_update_count is used to allow the userspace gettimeofday code
  410. * to assure itself that it sees a consistent view of the tb_to_xs and
  411. * stamp_xsec variables. It reads the tb_update_count, then reads
  412. * tb_to_xs and stamp_xsec and then reads tb_update_count again. If
  413. * the two values of tb_update_count match and are even then the
  414. * tb_to_xs and stamp_xsec values are consistent. If not, then it
  415. * loops back and reads them again until this criteria is met.
  416. * We expect the caller to have done the first increment of
  417. * vdso_data->tb_update_count already.
  418. */
  419. vdso_data->tb_orig_stamp = new_tb_stamp;
  420. vdso_data->stamp_xsec = new_stamp_xsec;
  421. vdso_data->tb_to_xs = new_tb_to_xs;
  422. vdso_data->wtom_clock_sec = wall_to_monotonic.tv_sec;
  423. vdso_data->wtom_clock_nsec = wall_to_monotonic.tv_nsec;
  424. smp_wmb();
  425. ++(vdso_data->tb_update_count);
  426. }
  427. /*
  428. * When the timebase - tb_orig_stamp gets too big, we do a manipulation
  429. * between tb_orig_stamp and stamp_xsec. The goal here is to keep the
  430. * difference tb - tb_orig_stamp small enough to always fit inside a
  431. * 32 bits number. This is a requirement of our fast 32 bits userland
  432. * implementation in the vdso. If we "miss" a call to this function
  433. * (interrupt latency, CPU locked in a spinlock, ...) and we end up
  434. * with a too big difference, then the vdso will fallback to calling
  435. * the syscall
  436. */
  437. static __inline__ void timer_recalc_offset(u64 cur_tb)
  438. {
  439. unsigned long offset;
  440. u64 new_stamp_xsec;
  441. u64 tlen, t2x;
  442. u64 tb, xsec_old, xsec_new;
  443. struct gettimeofday_vars *varp;
  444. if (__USE_RTC())
  445. return;
  446. tlen = current_tick_length();
  447. offset = cur_tb - do_gtod.varp->tb_orig_stamp;
  448. if (tlen == last_tick_len && offset < 0x80000000u)
  449. return;
  450. if (tlen != last_tick_len) {
  451. t2x = mulhdu(tlen << TICKLEN_SHIFT, ticklen_to_xs);
  452. last_tick_len = tlen;
  453. } else
  454. t2x = do_gtod.varp->tb_to_xs;
  455. new_stamp_xsec = (u64) xtime.tv_nsec * XSEC_PER_SEC;
  456. do_div(new_stamp_xsec, 1000000000);
  457. new_stamp_xsec += (u64) xtime.tv_sec * XSEC_PER_SEC;
  458. ++vdso_data->tb_update_count;
  459. smp_mb();
  460. /*
  461. * Make sure time doesn't go backwards for userspace gettimeofday.
  462. */
  463. tb = get_tb();
  464. varp = do_gtod.varp;
  465. xsec_old = mulhdu(tb - varp->tb_orig_stamp, varp->tb_to_xs)
  466. + varp->stamp_xsec;
  467. xsec_new = mulhdu(tb - cur_tb, t2x) + new_stamp_xsec;
  468. if (xsec_new < xsec_old)
  469. new_stamp_xsec += xsec_old - xsec_new;
  470. update_gtod(cur_tb, new_stamp_xsec, t2x);
  471. }
  472. #ifdef CONFIG_SMP
  473. unsigned long profile_pc(struct pt_regs *regs)
  474. {
  475. unsigned long pc = instruction_pointer(regs);
  476. if (in_lock_functions(pc))
  477. return regs->link;
  478. return pc;
  479. }
  480. EXPORT_SYMBOL(profile_pc);
  481. #endif
  482. #ifdef CONFIG_PPC_ISERIES
  483. /*
  484. * This function recalibrates the timebase based on the 49-bit time-of-day
  485. * value in the Titan chip. The Titan is much more accurate than the value
  486. * returned by the service processor for the timebase frequency.
  487. */
  488. static void iSeries_tb_recal(void)
  489. {
  490. struct div_result divres;
  491. unsigned long titan, tb;
  492. tb = get_tb();
  493. titan = HvCallXm_loadTod();
  494. if ( iSeries_recal_titan ) {
  495. unsigned long tb_ticks = tb - iSeries_recal_tb;
  496. unsigned long titan_usec = (titan - iSeries_recal_titan) >> 12;
  497. unsigned long new_tb_ticks_per_sec = (tb_ticks * USEC_PER_SEC)/titan_usec;
  498. unsigned long new_tb_ticks_per_jiffy = (new_tb_ticks_per_sec+(HZ/2))/HZ;
  499. long tick_diff = new_tb_ticks_per_jiffy - tb_ticks_per_jiffy;
  500. char sign = '+';
  501. /* make sure tb_ticks_per_sec and tb_ticks_per_jiffy are consistent */
  502. new_tb_ticks_per_sec = new_tb_ticks_per_jiffy * HZ;
  503. if ( tick_diff < 0 ) {
  504. tick_diff = -tick_diff;
  505. sign = '-';
  506. }
  507. if ( tick_diff ) {
  508. if ( tick_diff < tb_ticks_per_jiffy/25 ) {
  509. printk( "Titan recalibrate: new tb_ticks_per_jiffy = %lu (%c%ld)\n",
  510. new_tb_ticks_per_jiffy, sign, tick_diff );
  511. tb_ticks_per_jiffy = new_tb_ticks_per_jiffy;
  512. tb_ticks_per_sec = new_tb_ticks_per_sec;
  513. calc_cputime_factors();
  514. div128_by_32( XSEC_PER_SEC, 0, tb_ticks_per_sec, &divres );
  515. do_gtod.tb_ticks_per_sec = tb_ticks_per_sec;
  516. tb_to_xs = divres.result_low;
  517. do_gtod.varp->tb_to_xs = tb_to_xs;
  518. vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;
  519. vdso_data->tb_to_xs = tb_to_xs;
  520. }
  521. else {
  522. printk( "Titan recalibrate: FAILED (difference > 4 percent)\n"
  523. " new tb_ticks_per_jiffy = %lu\n"
  524. " old tb_ticks_per_jiffy = %lu\n",
  525. new_tb_ticks_per_jiffy, tb_ticks_per_jiffy );
  526. }
  527. }
  528. }
  529. iSeries_recal_titan = titan;
  530. iSeries_recal_tb = tb;
  531. }
  532. #endif
  533. /*
  534. * For iSeries shared processors, we have to let the hypervisor
  535. * set the hardware decrementer. We set a virtual decrementer
  536. * in the lppaca and call the hypervisor if the virtual
  537. * decrementer is less than the current value in the hardware
  538. * decrementer. (almost always the new decrementer value will
  539. * be greater than the current hardware decementer so the hypervisor
  540. * call will not be needed)
  541. */
  542. /*
  543. * timer_interrupt - gets called when the decrementer overflows,
  544. * with interrupts disabled.
  545. */
  546. void timer_interrupt(struct pt_regs * regs)
  547. {
  548. struct pt_regs *old_regs;
  549. int next_dec;
  550. int cpu = smp_processor_id();
  551. unsigned long ticks;
  552. u64 tb_next_jiffy;
  553. #ifdef CONFIG_PPC32
  554. if (atomic_read(&ppc_n_lost_interrupts) != 0)
  555. do_IRQ(regs);
  556. #endif
  557. old_regs = set_irq_regs(regs);
  558. irq_enter();
  559. profile_tick(CPU_PROFILING);
  560. calculate_steal_time();
  561. #ifdef CONFIG_PPC_ISERIES
  562. get_lppaca()->int_dword.fields.decr_int = 0;
  563. #endif
  564. while ((ticks = tb_ticks_since(per_cpu(last_jiffy, cpu)))
  565. >= tb_ticks_per_jiffy) {
  566. /* Update last_jiffy */
  567. per_cpu(last_jiffy, cpu) += tb_ticks_per_jiffy;
  568. /* Handle RTCL overflow on 601 */
  569. if (__USE_RTC() && per_cpu(last_jiffy, cpu) >= 1000000000)
  570. per_cpu(last_jiffy, cpu) -= 1000000000;
  571. /*
  572. * We cannot disable the decrementer, so in the period
  573. * between this cpu's being marked offline in cpu_online_map
  574. * and calling stop-self, it is taking timer interrupts.
  575. * Avoid calling into the scheduler rebalancing code if this
  576. * is the case.
  577. */
  578. if (!cpu_is_offline(cpu))
  579. account_process_time(regs);
  580. /*
  581. * No need to check whether cpu is offline here; boot_cpuid
  582. * should have been fixed up by now.
  583. */
  584. if (cpu != boot_cpuid)
  585. continue;
  586. write_seqlock(&xtime_lock);
  587. tb_next_jiffy = tb_last_jiffy + tb_ticks_per_jiffy;
  588. if (per_cpu(last_jiffy, cpu) >= tb_next_jiffy) {
  589. tb_last_jiffy = tb_next_jiffy;
  590. do_timer(1);
  591. timer_recalc_offset(tb_last_jiffy);
  592. timer_check_rtc();
  593. }
  594. write_sequnlock(&xtime_lock);
  595. }
  596. next_dec = tb_ticks_per_jiffy - ticks;
  597. set_dec(next_dec);
  598. #ifdef CONFIG_PPC_ISERIES
  599. if (hvlpevent_is_pending())
  600. process_hvlpevents();
  601. #endif
  602. #ifdef CONFIG_PPC64
  603. /* collect purr register values often, for accurate calculations */
  604. if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
  605. struct cpu_usage *cu = &__get_cpu_var(cpu_usage_array);
  606. cu->current_tb = mfspr(SPRN_PURR);
  607. }
  608. #endif
  609. irq_exit();
  610. set_irq_regs(old_regs);
  611. }
  612. void wakeup_decrementer(void)
  613. {
  614. unsigned long ticks;
  615. /*
  616. * The timebase gets saved on sleep and restored on wakeup,
  617. * so all we need to do is to reset the decrementer.
  618. */
  619. ticks = tb_ticks_since(__get_cpu_var(last_jiffy));
  620. if (ticks < tb_ticks_per_jiffy)
  621. ticks = tb_ticks_per_jiffy - ticks;
  622. else
  623. ticks = 1;
  624. set_dec(ticks);
  625. }
  626. #ifdef CONFIG_SMP
  627. void __init smp_space_timers(unsigned int max_cpus)
  628. {
  629. int i;
  630. unsigned long half = tb_ticks_per_jiffy / 2;
  631. unsigned long offset = tb_ticks_per_jiffy / max_cpus;
  632. u64 previous_tb = per_cpu(last_jiffy, boot_cpuid);
  633. /* make sure tb > per_cpu(last_jiffy, cpu) for all cpus always */
  634. previous_tb -= tb_ticks_per_jiffy;
  635. /*
  636. * The stolen time calculation for POWER5 shared-processor LPAR
  637. * systems works better if the two threads' timebase interrupts
  638. * are staggered by half a jiffy with respect to each other.
  639. */
  640. for_each_possible_cpu(i) {
  641. if (i == boot_cpuid)
  642. continue;
  643. if (i == (boot_cpuid ^ 1))
  644. per_cpu(last_jiffy, i) =
  645. per_cpu(last_jiffy, boot_cpuid) - half;
  646. else if (i & 1)
  647. per_cpu(last_jiffy, i) =
  648. per_cpu(last_jiffy, i ^ 1) + half;
  649. else {
  650. previous_tb += offset;
  651. per_cpu(last_jiffy, i) = previous_tb;
  652. }
  653. }
  654. }
  655. #endif
  656. /*
  657. * Scheduler clock - returns current time in nanosec units.
  658. *
  659. * Note: mulhdu(a, b) (multiply high double unsigned) returns
  660. * the high 64 bits of a * b, i.e. (a * b) >> 64, where a and b
  661. * are 64-bit unsigned numbers.
  662. */
  663. unsigned long long sched_clock(void)
  664. {
  665. if (__USE_RTC())
  666. return get_rtc();
  667. return mulhdu(get_tb(), tb_to_ns_scale) << tb_to_ns_shift;
  668. }
  669. int do_settimeofday(struct timespec *tv)
  670. {
  671. time_t wtm_sec, new_sec = tv->tv_sec;
  672. long wtm_nsec, new_nsec = tv->tv_nsec;
  673. unsigned long flags;
  674. u64 new_xsec;
  675. unsigned long tb_delta;
  676. if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
  677. return -EINVAL;
  678. write_seqlock_irqsave(&xtime_lock, flags);
  679. /*
  680. * Updating the RTC is not the job of this code. If the time is
  681. * stepped under NTP, the RTC will be updated after STA_UNSYNC
  682. * is cleared. Tools like clock/hwclock either copy the RTC
  683. * to the system time, in which case there is no point in writing
  684. * to the RTC again, or write to the RTC but then they don't call
  685. * settimeofday to perform this operation.
  686. */
  687. #ifdef CONFIG_PPC_ISERIES
  688. if (first_settimeofday) {
  689. iSeries_tb_recal();
  690. first_settimeofday = 0;
  691. }
  692. #endif
  693. /* Make userspace gettimeofday spin until we're done. */
  694. ++vdso_data->tb_update_count;
  695. smp_mb();
  696. /*
  697. * Subtract off the number of nanoseconds since the
  698. * beginning of the last tick.
  699. */
  700. tb_delta = tb_ticks_since(tb_last_jiffy);
  701. tb_delta = mulhdu(tb_delta, do_gtod.varp->tb_to_xs); /* in xsec */
  702. new_nsec -= SCALE_XSEC(tb_delta, 1000000000);
  703. wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - new_sec);
  704. wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - new_nsec);
  705. set_normalized_timespec(&xtime, new_sec, new_nsec);
  706. set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec);
  707. /* In case of a large backwards jump in time with NTP, we want the
  708. * clock to be updated as soon as the PLL is again in lock.
  709. */
  710. last_rtc_update = new_sec - 658;
  711. ntp_clear();
  712. new_xsec = xtime.tv_nsec;
  713. if (new_xsec != 0) {
  714. new_xsec *= XSEC_PER_SEC;
  715. do_div(new_xsec, NSEC_PER_SEC);
  716. }
  717. new_xsec += (u64)xtime.tv_sec * XSEC_PER_SEC;
  718. update_gtod(tb_last_jiffy, new_xsec, do_gtod.varp->tb_to_xs);
  719. vdso_data->tz_minuteswest = sys_tz.tz_minuteswest;
  720. vdso_data->tz_dsttime = sys_tz.tz_dsttime;
  721. write_sequnlock_irqrestore(&xtime_lock, flags);
  722. clock_was_set();
  723. return 0;
  724. }
  725. EXPORT_SYMBOL(do_settimeofday);
  726. static int __init get_freq(char *name, int cells, unsigned long *val)
  727. {
  728. struct device_node *cpu;
  729. const unsigned int *fp;
  730. int found = 0;
  731. /* The cpu node should have timebase and clock frequency properties */
  732. cpu = of_find_node_by_type(NULL, "cpu");
  733. if (cpu) {
  734. fp = get_property(cpu, name, NULL);
  735. if (fp) {
  736. found = 1;
  737. *val = of_read_ulong(fp, cells);
  738. }
  739. of_node_put(cpu);
  740. }
  741. return found;
  742. }
  743. void __init generic_calibrate_decr(void)
  744. {
  745. ppc_tb_freq = DEFAULT_TB_FREQ; /* hardcoded default */
  746. if (!get_freq("ibm,extended-timebase-frequency", 2, &ppc_tb_freq) &&
  747. !get_freq("timebase-frequency", 1, &ppc_tb_freq)) {
  748. printk(KERN_ERR "WARNING: Estimating decrementer frequency "
  749. "(not found)\n");
  750. }
  751. ppc_proc_freq = DEFAULT_PROC_FREQ; /* hardcoded default */
  752. if (!get_freq("ibm,extended-clock-frequency", 2, &ppc_proc_freq) &&
  753. !get_freq("clock-frequency", 1, &ppc_proc_freq)) {
  754. printk(KERN_ERR "WARNING: Estimating processor frequency "
  755. "(not found)\n");
  756. }
  757. #ifdef CONFIG_BOOKE
  758. /* Set the time base to zero */
  759. mtspr(SPRN_TBWL, 0);
  760. mtspr(SPRN_TBWU, 0);
  761. /* Clear any pending timer interrupts */
  762. mtspr(SPRN_TSR, TSR_ENW | TSR_WIS | TSR_DIS | TSR_FIS);
  763. /* Enable decrementer interrupt */
  764. mtspr(SPRN_TCR, TCR_DIE);
  765. #endif
  766. }
  767. unsigned long get_boot_time(void)
  768. {
  769. struct rtc_time tm;
  770. if (ppc_md.get_boot_time)
  771. return ppc_md.get_boot_time();
  772. if (!ppc_md.get_rtc_time)
  773. return 0;
  774. ppc_md.get_rtc_time(&tm);
  775. return mktime(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday,
  776. tm.tm_hour, tm.tm_min, tm.tm_sec);
  777. }
  778. /* This function is only called on the boot processor */
  779. void __init time_init(void)
  780. {
  781. unsigned long flags;
  782. unsigned long tm = 0;
  783. struct div_result res;
  784. u64 scale, x;
  785. unsigned shift;
  786. if (ppc_md.time_init != NULL)
  787. timezone_offset = ppc_md.time_init();
  788. if (__USE_RTC()) {
  789. /* 601 processor: dec counts down by 128 every 128ns */
  790. ppc_tb_freq = 1000000000;
  791. tb_last_jiffy = get_rtcl();
  792. } else {
  793. /* Normal PowerPC with timebase register */
  794. ppc_md.calibrate_decr();
  795. printk(KERN_DEBUG "time_init: decrementer frequency = %lu.%.6lu MHz\n",
  796. ppc_tb_freq / 1000000, ppc_tb_freq % 1000000);
  797. printk(KERN_DEBUG "time_init: processor frequency = %lu.%.6lu MHz\n",
  798. ppc_proc_freq / 1000000, ppc_proc_freq % 1000000);
  799. tb_last_jiffy = get_tb();
  800. }
  801. tb_ticks_per_jiffy = ppc_tb_freq / HZ;
  802. tb_ticks_per_sec = ppc_tb_freq;
  803. tb_ticks_per_usec = ppc_tb_freq / 1000000;
  804. tb_to_us = mulhwu_scale_factor(ppc_tb_freq, 1000000);
  805. calc_cputime_factors();
  806. /*
  807. * Calculate the length of each tick in ns. It will not be
  808. * exactly 1e9/HZ unless ppc_tb_freq is divisible by HZ.
  809. * We compute 1e9 * tb_ticks_per_jiffy / ppc_tb_freq,
  810. * rounded up.
  811. */
  812. x = (u64) NSEC_PER_SEC * tb_ticks_per_jiffy + ppc_tb_freq - 1;
  813. do_div(x, ppc_tb_freq);
  814. tick_nsec = x;
  815. last_tick_len = x << TICKLEN_SCALE;
  816. /*
  817. * Compute ticklen_to_xs, which is a factor which gets multiplied
  818. * by (last_tick_len << TICKLEN_SHIFT) to get a tb_to_xs value.
  819. * It is computed as:
  820. * ticklen_to_xs = 2^N / (tb_ticks_per_jiffy * 1e9)
  821. * where N = 64 + 20 - TICKLEN_SCALE - TICKLEN_SHIFT
  822. * which turns out to be N = 51 - SHIFT_HZ.
  823. * This gives the result as a 0.64 fixed-point fraction.
  824. * That value is reduced by an offset amounting to 1 xsec per
  825. * 2^31 timebase ticks to avoid problems with time going backwards
  826. * by 1 xsec when we do timer_recalc_offset due to losing the
  827. * fractional xsec. That offset is equal to ppc_tb_freq/2^51
  828. * since there are 2^20 xsec in a second.
  829. */
  830. div128_by_32((1ULL << 51) - ppc_tb_freq, 0,
  831. tb_ticks_per_jiffy << SHIFT_HZ, &res);
  832. div128_by_32(res.result_high, res.result_low, NSEC_PER_SEC, &res);
  833. ticklen_to_xs = res.result_low;
  834. /* Compute tb_to_xs from tick_nsec */
  835. tb_to_xs = mulhdu(last_tick_len << TICKLEN_SHIFT, ticklen_to_xs);
  836. /*
  837. * Compute scale factor for sched_clock.
  838. * The calibrate_decr() function has set tb_ticks_per_sec,
  839. * which is the timebase frequency.
  840. * We compute 1e9 * 2^64 / tb_ticks_per_sec and interpret
  841. * the 128-bit result as a 64.64 fixed-point number.
  842. * We then shift that number right until it is less than 1.0,
  843. * giving us the scale factor and shift count to use in
  844. * sched_clock().
  845. */
  846. div128_by_32(1000000000, 0, tb_ticks_per_sec, &res);
  847. scale = res.result_low;
  848. for (shift = 0; res.result_high != 0; ++shift) {
  849. scale = (scale >> 1) | (res.result_high << 63);
  850. res.result_high >>= 1;
  851. }
  852. tb_to_ns_scale = scale;
  853. tb_to_ns_shift = shift;
  854. tm = get_boot_time();
  855. write_seqlock_irqsave(&xtime_lock, flags);
  856. /* If platform provided a timezone (pmac), we correct the time */
  857. if (timezone_offset) {
  858. sys_tz.tz_minuteswest = -timezone_offset / 60;
  859. sys_tz.tz_dsttime = 0;
  860. tm -= timezone_offset;
  861. }
  862. xtime.tv_sec = tm;
  863. xtime.tv_nsec = 0;
  864. do_gtod.varp = &do_gtod.vars[0];
  865. do_gtod.var_idx = 0;
  866. do_gtod.varp->tb_orig_stamp = tb_last_jiffy;
  867. __get_cpu_var(last_jiffy) = tb_last_jiffy;
  868. do_gtod.varp->stamp_xsec = (u64) xtime.tv_sec * XSEC_PER_SEC;
  869. do_gtod.tb_ticks_per_sec = tb_ticks_per_sec;
  870. do_gtod.varp->tb_to_xs = tb_to_xs;
  871. do_gtod.tb_to_us = tb_to_us;
  872. vdso_data->tb_orig_stamp = tb_last_jiffy;
  873. vdso_data->tb_update_count = 0;
  874. vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;
  875. vdso_data->stamp_xsec = (u64) xtime.tv_sec * XSEC_PER_SEC;
  876. vdso_data->tb_to_xs = tb_to_xs;
  877. time_freq = 0;
  878. last_rtc_update = xtime.tv_sec;
  879. set_normalized_timespec(&wall_to_monotonic,
  880. -xtime.tv_sec, -xtime.tv_nsec);
  881. write_sequnlock_irqrestore(&xtime_lock, flags);
  882. /* Not exact, but the timer interrupt takes care of this */
  883. set_dec(tb_ticks_per_jiffy);
  884. }
  885. #ifdef CONFIG_RTC_CLASS
  886. static int set_rtc_class_time(struct rtc_time *tm)
  887. {
  888. int err;
  889. struct class_device *class_dev =
  890. rtc_class_open(CONFIG_RTC_HCTOSYS_DEVICE);
  891. if (class_dev == NULL)
  892. return -ENODEV;
  893. err = rtc_set_time(class_dev, tm);
  894. rtc_class_close(class_dev);
  895. return 0;
  896. }
  897. static void get_rtc_class_time(struct rtc_time *tm)
  898. {
  899. int err;
  900. struct class_device *class_dev =
  901. rtc_class_open(CONFIG_RTC_HCTOSYS_DEVICE);
  902. if (class_dev == NULL)
  903. return;
  904. err = rtc_read_time(class_dev, tm);
  905. rtc_class_close(class_dev);
  906. return;
  907. }
  908. int __init rtc_class_hookup(void)
  909. {
  910. ppc_md.get_rtc_time = get_rtc_class_time;
  911. ppc_md.set_rtc_time = set_rtc_class_time;
  912. return 0;
  913. }
  914. #endif /* CONFIG_RTC_CLASS */
  915. #define FEBRUARY 2
  916. #define STARTOFTIME 1970
  917. #define SECDAY 86400L
  918. #define SECYR (SECDAY * 365)
  919. #define leapyear(year) ((year) % 4 == 0 && \
  920. ((year) % 100 != 0 || (year) % 400 == 0))
  921. #define days_in_year(a) (leapyear(a) ? 366 : 365)
  922. #define days_in_month(a) (month_days[(a) - 1])
  923. static int month_days[12] = {
  924. 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
  925. };
  926. /*
  927. * This only works for the Gregorian calendar - i.e. after 1752 (in the UK)
  928. */
  929. void GregorianDay(struct rtc_time * tm)
  930. {
  931. int leapsToDate;
  932. int lastYear;
  933. int day;
  934. int MonthOffset[] = { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 };
  935. lastYear = tm->tm_year - 1;
  936. /*
  937. * Number of leap corrections to apply up to end of last year
  938. */
  939. leapsToDate = lastYear / 4 - lastYear / 100 + lastYear / 400;
  940. /*
  941. * This year is a leap year if it is divisible by 4 except when it is
  942. * divisible by 100 unless it is divisible by 400
  943. *
  944. * e.g. 1904 was a leap year, 1900 was not, 1996 is, and 2000 was
  945. */
  946. day = tm->tm_mon > 2 && leapyear(tm->tm_year);
  947. day += lastYear*365 + leapsToDate + MonthOffset[tm->tm_mon-1] +
  948. tm->tm_mday;
  949. tm->tm_wday = day % 7;
  950. }
  951. void to_tm(int tim, struct rtc_time * tm)
  952. {
  953. register int i;
  954. register long hms, day;
  955. day = tim / SECDAY;
  956. hms = tim % SECDAY;
  957. /* Hours, minutes, seconds are easy */
  958. tm->tm_hour = hms / 3600;
  959. tm->tm_min = (hms % 3600) / 60;
  960. tm->tm_sec = (hms % 3600) % 60;
  961. /* Number of years in days */
  962. for (i = STARTOFTIME; day >= days_in_year(i); i++)
  963. day -= days_in_year(i);
  964. tm->tm_year = i;
  965. /* Number of months in days left */
  966. if (leapyear(tm->tm_year))
  967. days_in_month(FEBRUARY) = 29;
  968. for (i = 1; day >= days_in_month(i); i++)
  969. day -= days_in_month(i);
  970. days_in_month(FEBRUARY) = 28;
  971. tm->tm_mon = i;
  972. /* Days are what is left over (+1) from all that. */
  973. tm->tm_mday = day + 1;
  974. /*
  975. * Determine the day of week
  976. */
  977. GregorianDay(tm);
  978. }
  979. /* Auxiliary function to compute scaling factors */
  980. /* Actually the choice of a timebase running at 1/4 the of the bus
  981. * frequency giving resolution of a few tens of nanoseconds is quite nice.
  982. * It makes this computation very precise (27-28 bits typically) which
  983. * is optimistic considering the stability of most processor clock
  984. * oscillators and the precision with which the timebase frequency
  985. * is measured but does not harm.
  986. */
  987. unsigned mulhwu_scale_factor(unsigned inscale, unsigned outscale)
  988. {
  989. unsigned mlt=0, tmp, err;
  990. /* No concern for performance, it's done once: use a stupid
  991. * but safe and compact method to find the multiplier.
  992. */
  993. for (tmp = 1U<<31; tmp != 0; tmp >>= 1) {
  994. if (mulhwu(inscale, mlt|tmp) < outscale)
  995. mlt |= tmp;
  996. }
  997. /* We might still be off by 1 for the best approximation.
  998. * A side effect of this is that if outscale is too large
  999. * the returned value will be zero.
  1000. * Many corner cases have been checked and seem to work,
  1001. * some might have been forgotten in the test however.
  1002. */
  1003. err = inscale * (mlt+1);
  1004. if (err <= inscale/2)
  1005. mlt++;
  1006. return mlt;
  1007. }
  1008. /*
  1009. * Divide a 128-bit dividend by a 32-bit divisor, leaving a 128 bit
  1010. * result.
  1011. */
  1012. void div128_by_32(u64 dividend_high, u64 dividend_low,
  1013. unsigned divisor, struct div_result *dr)
  1014. {
  1015. unsigned long a, b, c, d;
  1016. unsigned long w, x, y, z;
  1017. u64 ra, rb, rc;
  1018. a = dividend_high >> 32;
  1019. b = dividend_high & 0xffffffff;
  1020. c = dividend_low >> 32;
  1021. d = dividend_low & 0xffffffff;
  1022. w = a / divisor;
  1023. ra = ((u64)(a - (w * divisor)) << 32) + b;
  1024. rb = ((u64) do_div(ra, divisor) << 32) + c;
  1025. x = ra;
  1026. rc = ((u64) do_div(rb, divisor) << 32) + d;
  1027. y = rb;
  1028. do_div(rc, divisor);
  1029. z = rc;
  1030. dr->result_high = ((u64)w << 32) + x;
  1031. dr->result_low = ((u64)y << 32) + z;
  1032. }