process_32.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875
  1. /*
  2. * Copyright (C) 1995 Linus Torvalds
  3. *
  4. * Pentium III FXSR, SSE support
  5. * Gareth Hughes <gareth@valinux.com>, May 2000
  6. */
  7. /*
  8. * This file handles the architecture-dependent parts of process handling..
  9. */
  10. #include <stdarg.h>
  11. #include <linux/cpu.h>
  12. #include <linux/errno.h>
  13. #include <linux/sched.h>
  14. #include <linux/fs.h>
  15. #include <linux/kernel.h>
  16. #include <linux/mm.h>
  17. #include <linux/elfcore.h>
  18. #include <linux/smp.h>
  19. #include <linux/stddef.h>
  20. #include <linux/slab.h>
  21. #include <linux/vmalloc.h>
  22. #include <linux/user.h>
  23. #include <linux/interrupt.h>
  24. #include <linux/utsname.h>
  25. #include <linux/delay.h>
  26. #include <linux/reboot.h>
  27. #include <linux/init.h>
  28. #include <linux/mc146818rtc.h>
  29. #include <linux/module.h>
  30. #include <linux/kallsyms.h>
  31. #include <linux/ptrace.h>
  32. #include <linux/random.h>
  33. #include <linux/personality.h>
  34. #include <linux/tick.h>
  35. #include <linux/percpu.h>
  36. #include <linux/prctl.h>
  37. #include <asm/uaccess.h>
  38. #include <asm/pgtable.h>
  39. #include <asm/system.h>
  40. #include <asm/io.h>
  41. #include <asm/ldt.h>
  42. #include <asm/processor.h>
  43. #include <asm/i387.h>
  44. #include <asm/desc.h>
  45. #ifdef CONFIG_MATH_EMULATION
  46. #include <asm/math_emu.h>
  47. #endif
  48. #include <linux/err.h>
  49. #include <asm/tlbflush.h>
  50. #include <asm/cpu.h>
  51. #include <asm/kdebug.h>
  52. asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
  53. static int hlt_counter;
  54. unsigned long boot_option_idle_override = 0;
  55. EXPORT_SYMBOL(boot_option_idle_override);
  56. DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
  57. EXPORT_PER_CPU_SYMBOL(current_task);
  58. DEFINE_PER_CPU(int, cpu_number);
  59. EXPORT_PER_CPU_SYMBOL(cpu_number);
  60. /*
  61. * Return saved PC of a blocked thread.
  62. */
  63. unsigned long thread_saved_pc(struct task_struct *tsk)
  64. {
  65. return ((unsigned long *)tsk->thread.sp)[3];
  66. }
  67. /*
  68. * Powermanagement idle function, if any..
  69. */
  70. void (*pm_idle)(void);
  71. EXPORT_SYMBOL(pm_idle);
  72. void disable_hlt(void)
  73. {
  74. hlt_counter++;
  75. }
  76. EXPORT_SYMBOL(disable_hlt);
  77. void enable_hlt(void)
  78. {
  79. hlt_counter--;
  80. }
  81. EXPORT_SYMBOL(enable_hlt);
  82. /*
  83. * We use this if we don't have any better
  84. * idle routine..
  85. */
  86. void default_idle(void)
  87. {
  88. if (!hlt_counter && boot_cpu_data.hlt_works_ok) {
  89. current_thread_info()->status &= ~TS_POLLING;
  90. /*
  91. * TS_POLLING-cleared state must be visible before we
  92. * test NEED_RESCHED:
  93. */
  94. smp_mb();
  95. local_irq_disable();
  96. if (!need_resched()) {
  97. safe_halt(); /* enables interrupts racelessly */
  98. local_irq_disable();
  99. }
  100. local_irq_enable();
  101. current_thread_info()->status |= TS_POLLING;
  102. } else {
  103. local_irq_enable();
  104. /* loop is done by the caller */
  105. cpu_relax();
  106. }
  107. }
  108. #ifdef CONFIG_APM_MODULE
  109. EXPORT_SYMBOL(default_idle);
  110. #endif
  111. /*
  112. * On SMP it's slightly faster (but much more power-consuming!)
  113. * to poll the ->work.need_resched flag instead of waiting for the
  114. * cross-CPU IPI to arrive. Use this option with caution.
  115. */
  116. static void poll_idle(void)
  117. {
  118. local_irq_enable();
  119. cpu_relax();
  120. }
  121. #ifdef CONFIG_HOTPLUG_CPU
  122. #include <asm/nmi.h>
  123. /* We don't actually take CPU down, just spin without interrupts. */
  124. static inline void play_dead(void)
  125. {
  126. /* This must be done before dead CPU ack */
  127. cpu_exit_clear();
  128. wbinvd();
  129. mb();
  130. /* Ack it */
  131. __get_cpu_var(cpu_state) = CPU_DEAD;
  132. /*
  133. * With physical CPU hotplug, we should halt the cpu
  134. */
  135. local_irq_disable();
  136. while (1)
  137. halt();
  138. }
  139. #else
  140. static inline void play_dead(void)
  141. {
  142. BUG();
  143. }
  144. #endif /* CONFIG_HOTPLUG_CPU */
  145. /*
  146. * The idle thread. There's no useful work to be
  147. * done, so just try to conserve power and have a
  148. * low exit latency (ie sit in a loop waiting for
  149. * somebody to say that they'd like to reschedule)
  150. */
  151. void cpu_idle(void)
  152. {
  153. int cpu = smp_processor_id();
  154. current_thread_info()->status |= TS_POLLING;
  155. /* endless idle loop with no priority at all */
  156. while (1) {
  157. tick_nohz_stop_sched_tick();
  158. while (!need_resched()) {
  159. void (*idle)(void);
  160. check_pgt_cache();
  161. rmb();
  162. idle = pm_idle;
  163. if (rcu_pending(cpu))
  164. rcu_check_callbacks(cpu, 0);
  165. if (!idle)
  166. idle = default_idle;
  167. if (cpu_is_offline(cpu))
  168. play_dead();
  169. __get_cpu_var(irq_stat).idle_timestamp = jiffies;
  170. idle();
  171. }
  172. tick_nohz_restart_sched_tick();
  173. preempt_enable_no_resched();
  174. schedule();
  175. preempt_disable();
  176. }
  177. }
  178. static void do_nothing(void *unused)
  179. {
  180. }
  181. /*
  182. * cpu_idle_wait - Used to ensure that all the CPUs discard old value of
  183. * pm_idle and update to new pm_idle value. Required while changing pm_idle
  184. * handler on SMP systems.
  185. *
  186. * Caller must have changed pm_idle to the new value before the call. Old
  187. * pm_idle value will not be used by any CPU after the return of this function.
  188. */
  189. void cpu_idle_wait(void)
  190. {
  191. smp_mb();
  192. /* kick all the CPUs so that they exit out of pm_idle */
  193. smp_call_function(do_nothing, NULL, 0, 1);
  194. }
  195. EXPORT_SYMBOL_GPL(cpu_idle_wait);
  196. /*
  197. * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
  198. * which can obviate IPI to trigger checking of need_resched.
  199. * We execute MONITOR against need_resched and enter optimized wait state
  200. * through MWAIT. Whenever someone changes need_resched, we would be woken
  201. * up from MWAIT (without an IPI).
  202. *
  203. * New with Core Duo processors, MWAIT can take some hints based on CPU
  204. * capability.
  205. */
  206. void mwait_idle_with_hints(unsigned long ax, unsigned long cx)
  207. {
  208. if (!need_resched()) {
  209. __monitor((void *)&current_thread_info()->flags, 0, 0);
  210. smp_mb();
  211. if (!need_resched())
  212. __sti_mwait(ax, cx);
  213. else
  214. local_irq_enable();
  215. } else
  216. local_irq_enable();
  217. }
  218. /* Default MONITOR/MWAIT with no hints, used for default C1 state */
  219. static void mwait_idle(void)
  220. {
  221. local_irq_enable();
  222. mwait_idle_with_hints(0, 0);
  223. }
  224. static int __cpuinit mwait_usable(const struct cpuinfo_x86 *c)
  225. {
  226. if (force_mwait)
  227. return 1;
  228. /* Any C1 states supported? */
  229. return c->cpuid_level >= 5 && ((cpuid_edx(5) >> 4) & 0xf) > 0;
  230. }
  231. void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
  232. {
  233. static int selected;
  234. if (selected)
  235. return;
  236. #ifdef CONFIG_X86_SMP
  237. if (pm_idle == poll_idle && smp_num_siblings > 1) {
  238. printk(KERN_WARNING "WARNING: polling idle and HT enabled,"
  239. " performance may degrade.\n");
  240. }
  241. #endif
  242. if (cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) {
  243. /*
  244. * Skip, if setup has overridden idle.
  245. * One CPU supports mwait => All CPUs supports mwait
  246. */
  247. if (!pm_idle) {
  248. printk(KERN_INFO "using mwait in idle threads.\n");
  249. pm_idle = mwait_idle;
  250. }
  251. }
  252. selected = 1;
  253. }
  254. static int __init idle_setup(char *str)
  255. {
  256. if (!strcmp(str, "poll")) {
  257. printk("using polling idle threads.\n");
  258. pm_idle = poll_idle;
  259. } else if (!strcmp(str, "mwait"))
  260. force_mwait = 1;
  261. else
  262. return -1;
  263. boot_option_idle_override = 1;
  264. return 0;
  265. }
  266. early_param("idle", idle_setup);
  267. void __show_registers(struct pt_regs *regs, int all)
  268. {
  269. unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L;
  270. unsigned long d0, d1, d2, d3, d6, d7;
  271. unsigned long sp;
  272. unsigned short ss, gs;
  273. if (user_mode_vm(regs)) {
  274. sp = regs->sp;
  275. ss = regs->ss & 0xffff;
  276. savesegment(gs, gs);
  277. } else {
  278. sp = (unsigned long) (&regs->sp);
  279. savesegment(ss, ss);
  280. savesegment(gs, gs);
  281. }
  282. printk("\n");
  283. printk("Pid: %d, comm: %s %s (%s %.*s)\n",
  284. task_pid_nr(current), current->comm,
  285. print_tainted(), init_utsname()->release,
  286. (int)strcspn(init_utsname()->version, " "),
  287. init_utsname()->version);
  288. printk("EIP: %04x:[<%08lx>] EFLAGS: %08lx CPU: %d\n",
  289. (u16)regs->cs, regs->ip, regs->flags,
  290. smp_processor_id());
  291. print_symbol("EIP is at %s\n", regs->ip);
  292. printk("EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n",
  293. regs->ax, regs->bx, regs->cx, regs->dx);
  294. printk("ESI: %08lx EDI: %08lx EBP: %08lx ESP: %08lx\n",
  295. regs->si, regs->di, regs->bp, sp);
  296. printk(" DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x\n",
  297. (u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, ss);
  298. if (!all)
  299. return;
  300. cr0 = read_cr0();
  301. cr2 = read_cr2();
  302. cr3 = read_cr3();
  303. cr4 = read_cr4_safe();
  304. printk("CR0: %08lx CR2: %08lx CR3: %08lx CR4: %08lx\n",
  305. cr0, cr2, cr3, cr4);
  306. get_debugreg(d0, 0);
  307. get_debugreg(d1, 1);
  308. get_debugreg(d2, 2);
  309. get_debugreg(d3, 3);
  310. printk("DR0: %08lx DR1: %08lx DR2: %08lx DR3: %08lx\n",
  311. d0, d1, d2, d3);
  312. get_debugreg(d6, 6);
  313. get_debugreg(d7, 7);
  314. printk("DR6: %08lx DR7: %08lx\n",
  315. d6, d7);
  316. }
  317. void show_regs(struct pt_regs *regs)
  318. {
  319. __show_registers(regs, 1);
  320. show_trace(NULL, regs, &regs->sp, regs->bp);
  321. }
  322. /*
  323. * This gets run with %bx containing the
  324. * function to call, and %dx containing
  325. * the "args".
  326. */
  327. extern void kernel_thread_helper(void);
  328. /*
  329. * Create a kernel thread
  330. */
  331. int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
  332. {
  333. struct pt_regs regs;
  334. memset(&regs, 0, sizeof(regs));
  335. regs.bx = (unsigned long) fn;
  336. regs.dx = (unsigned long) arg;
  337. regs.ds = __USER_DS;
  338. regs.es = __USER_DS;
  339. regs.fs = __KERNEL_PERCPU;
  340. regs.orig_ax = -1;
  341. regs.ip = (unsigned long) kernel_thread_helper;
  342. regs.cs = __KERNEL_CS | get_kernel_rpl();
  343. regs.flags = X86_EFLAGS_IF | X86_EFLAGS_SF | X86_EFLAGS_PF | 0x2;
  344. /* Ok, create the new process.. */
  345. return do_fork(flags | CLONE_VM | CLONE_UNTRACED, 0, &regs, 0, NULL, NULL);
  346. }
  347. EXPORT_SYMBOL(kernel_thread);
  348. /*
  349. * Free current thread data structures etc..
  350. */
  351. void exit_thread(void)
  352. {
  353. /* The process may have allocated an io port bitmap... nuke it. */
  354. if (unlikely(test_thread_flag(TIF_IO_BITMAP))) {
  355. struct task_struct *tsk = current;
  356. struct thread_struct *t = &tsk->thread;
  357. int cpu = get_cpu();
  358. struct tss_struct *tss = &per_cpu(init_tss, cpu);
  359. kfree(t->io_bitmap_ptr);
  360. t->io_bitmap_ptr = NULL;
  361. clear_thread_flag(TIF_IO_BITMAP);
  362. /*
  363. * Careful, clear this in the TSS too:
  364. */
  365. memset(tss->io_bitmap, 0xff, tss->io_bitmap_max);
  366. t->io_bitmap_max = 0;
  367. tss->io_bitmap_owner = NULL;
  368. tss->io_bitmap_max = 0;
  369. tss->x86_tss.io_bitmap_base = INVALID_IO_BITMAP_OFFSET;
  370. put_cpu();
  371. }
  372. }
  373. void flush_thread(void)
  374. {
  375. struct task_struct *tsk = current;
  376. tsk->thread.debugreg0 = 0;
  377. tsk->thread.debugreg1 = 0;
  378. tsk->thread.debugreg2 = 0;
  379. tsk->thread.debugreg3 = 0;
  380. tsk->thread.debugreg6 = 0;
  381. tsk->thread.debugreg7 = 0;
  382. memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
  383. clear_tsk_thread_flag(tsk, TIF_DEBUG);
  384. /*
  385. * Forget coprocessor state..
  386. */
  387. clear_fpu(tsk);
  388. clear_used_math();
  389. }
  390. void release_thread(struct task_struct *dead_task)
  391. {
  392. BUG_ON(dead_task->mm);
  393. release_vm86_irqs(dead_task);
  394. }
  395. /*
  396. * This gets called before we allocate a new thread and copy
  397. * the current task into it.
  398. */
  399. void prepare_to_copy(struct task_struct *tsk)
  400. {
  401. unlazy_fpu(tsk);
  402. }
  403. int copy_thread(int nr, unsigned long clone_flags, unsigned long sp,
  404. unsigned long unused,
  405. struct task_struct * p, struct pt_regs * regs)
  406. {
  407. struct pt_regs * childregs;
  408. struct task_struct *tsk;
  409. int err;
  410. childregs = task_pt_regs(p);
  411. *childregs = *regs;
  412. childregs->ax = 0;
  413. childregs->sp = sp;
  414. p->thread.sp = (unsigned long) childregs;
  415. p->thread.sp0 = (unsigned long) (childregs+1);
  416. p->thread.ip = (unsigned long) ret_from_fork;
  417. savesegment(gs, p->thread.gs);
  418. tsk = current;
  419. if (unlikely(test_tsk_thread_flag(tsk, TIF_IO_BITMAP))) {
  420. p->thread.io_bitmap_ptr = kmemdup(tsk->thread.io_bitmap_ptr,
  421. IO_BITMAP_BYTES, GFP_KERNEL);
  422. if (!p->thread.io_bitmap_ptr) {
  423. p->thread.io_bitmap_max = 0;
  424. return -ENOMEM;
  425. }
  426. set_tsk_thread_flag(p, TIF_IO_BITMAP);
  427. }
  428. err = 0;
  429. /*
  430. * Set a new TLS for the child thread?
  431. */
  432. if (clone_flags & CLONE_SETTLS)
  433. err = do_set_thread_area(p, -1,
  434. (struct user_desc __user *)childregs->si, 0);
  435. if (err && p->thread.io_bitmap_ptr) {
  436. kfree(p->thread.io_bitmap_ptr);
  437. p->thread.io_bitmap_max = 0;
  438. }
  439. return err;
  440. }
  441. void
  442. start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
  443. {
  444. __asm__("movl %0, %%gs" :: "r"(0));
  445. regs->fs = 0;
  446. set_fs(USER_DS);
  447. regs->ds = __USER_DS;
  448. regs->es = __USER_DS;
  449. regs->ss = __USER_DS;
  450. regs->cs = __USER_CS;
  451. regs->ip = new_ip;
  452. regs->sp = new_sp;
  453. /*
  454. * Free the old FP and other extended state
  455. */
  456. free_thread_xstate(current);
  457. }
  458. EXPORT_SYMBOL_GPL(start_thread);
  459. static void hard_disable_TSC(void)
  460. {
  461. write_cr4(read_cr4() | X86_CR4_TSD);
  462. }
  463. void disable_TSC(void)
  464. {
  465. preempt_disable();
  466. if (!test_and_set_thread_flag(TIF_NOTSC))
  467. /*
  468. * Must flip the CPU state synchronously with
  469. * TIF_NOTSC in the current running context.
  470. */
  471. hard_disable_TSC();
  472. preempt_enable();
  473. }
  474. static void hard_enable_TSC(void)
  475. {
  476. write_cr4(read_cr4() & ~X86_CR4_TSD);
  477. }
  478. static void enable_TSC(void)
  479. {
  480. preempt_disable();
  481. if (test_and_clear_thread_flag(TIF_NOTSC))
  482. /*
  483. * Must flip the CPU state synchronously with
  484. * TIF_NOTSC in the current running context.
  485. */
  486. hard_enable_TSC();
  487. preempt_enable();
  488. }
  489. int get_tsc_mode(unsigned long adr)
  490. {
  491. unsigned int val;
  492. if (test_thread_flag(TIF_NOTSC))
  493. val = PR_TSC_SIGSEGV;
  494. else
  495. val = PR_TSC_ENABLE;
  496. return put_user(val, (unsigned int __user *)adr);
  497. }
  498. int set_tsc_mode(unsigned int val)
  499. {
  500. if (val == PR_TSC_SIGSEGV)
  501. disable_TSC();
  502. else if (val == PR_TSC_ENABLE)
  503. enable_TSC();
  504. else
  505. return -EINVAL;
  506. return 0;
  507. }
  508. static noinline void
  509. __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
  510. struct tss_struct *tss)
  511. {
  512. struct thread_struct *prev, *next;
  513. unsigned long debugctl;
  514. prev = &prev_p->thread;
  515. next = &next_p->thread;
  516. debugctl = prev->debugctlmsr;
  517. if (next->ds_area_msr != prev->ds_area_msr) {
  518. /* we clear debugctl to make sure DS
  519. * is not in use when we change it */
  520. debugctl = 0;
  521. update_debugctlmsr(0);
  522. wrmsr(MSR_IA32_DS_AREA, next->ds_area_msr, 0);
  523. }
  524. if (next->debugctlmsr != debugctl)
  525. update_debugctlmsr(next->debugctlmsr);
  526. if (test_tsk_thread_flag(next_p, TIF_DEBUG)) {
  527. set_debugreg(next->debugreg0, 0);
  528. set_debugreg(next->debugreg1, 1);
  529. set_debugreg(next->debugreg2, 2);
  530. set_debugreg(next->debugreg3, 3);
  531. /* no 4 and 5 */
  532. set_debugreg(next->debugreg6, 6);
  533. set_debugreg(next->debugreg7, 7);
  534. }
  535. if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
  536. test_tsk_thread_flag(next_p, TIF_NOTSC)) {
  537. /* prev and next are different */
  538. if (test_tsk_thread_flag(next_p, TIF_NOTSC))
  539. hard_disable_TSC();
  540. else
  541. hard_enable_TSC();
  542. }
  543. #ifdef X86_BTS
  544. if (test_tsk_thread_flag(prev_p, TIF_BTS_TRACE_TS))
  545. ptrace_bts_take_timestamp(prev_p, BTS_TASK_DEPARTS);
  546. if (test_tsk_thread_flag(next_p, TIF_BTS_TRACE_TS))
  547. ptrace_bts_take_timestamp(next_p, BTS_TASK_ARRIVES);
  548. #endif
  549. if (!test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
  550. /*
  551. * Disable the bitmap via an invalid offset. We still cache
  552. * the previous bitmap owner and the IO bitmap contents:
  553. */
  554. tss->x86_tss.io_bitmap_base = INVALID_IO_BITMAP_OFFSET;
  555. return;
  556. }
  557. if (likely(next == tss->io_bitmap_owner)) {
  558. /*
  559. * Previous owner of the bitmap (hence the bitmap content)
  560. * matches the next task, we dont have to do anything but
  561. * to set a valid offset in the TSS:
  562. */
  563. tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
  564. return;
  565. }
  566. /*
  567. * Lazy TSS's I/O bitmap copy. We set an invalid offset here
  568. * and we let the task to get a GPF in case an I/O instruction
  569. * is performed. The handler of the GPF will verify that the
  570. * faulting task has a valid I/O bitmap and, it true, does the
  571. * real copy and restart the instruction. This will save us
  572. * redundant copies when the currently switched task does not
  573. * perform any I/O during its timeslice.
  574. */
  575. tss->x86_tss.io_bitmap_base = INVALID_IO_BITMAP_OFFSET_LAZY;
  576. }
  577. /*
  578. * switch_to(x,yn) should switch tasks from x to y.
  579. *
  580. * We fsave/fwait so that an exception goes off at the right time
  581. * (as a call from the fsave or fwait in effect) rather than to
  582. * the wrong process. Lazy FP saving no longer makes any sense
  583. * with modern CPU's, and this simplifies a lot of things (SMP
  584. * and UP become the same).
  585. *
  586. * NOTE! We used to use the x86 hardware context switching. The
  587. * reason for not using it any more becomes apparent when you
  588. * try to recover gracefully from saved state that is no longer
  589. * valid (stale segment register values in particular). With the
  590. * hardware task-switch, there is no way to fix up bad state in
  591. * a reasonable manner.
  592. *
  593. * The fact that Intel documents the hardware task-switching to
  594. * be slow is a fairly red herring - this code is not noticeably
  595. * faster. However, there _is_ some room for improvement here,
  596. * so the performance issues may eventually be a valid point.
  597. * More important, however, is the fact that this allows us much
  598. * more flexibility.
  599. *
  600. * The return value (in %ax) will be the "prev" task after
  601. * the task-switch, and shows up in ret_from_fork in entry.S,
  602. * for example.
  603. */
  604. struct task_struct * __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
  605. {
  606. struct thread_struct *prev = &prev_p->thread,
  607. *next = &next_p->thread;
  608. int cpu = smp_processor_id();
  609. struct tss_struct *tss = &per_cpu(init_tss, cpu);
  610. /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
  611. __unlazy_fpu(prev_p);
  612. /* we're going to use this soon, after a few expensive things */
  613. if (next_p->fpu_counter > 5)
  614. prefetch(next->xstate);
  615. /*
  616. * Reload esp0.
  617. */
  618. load_sp0(tss, next);
  619. /*
  620. * Save away %gs. No need to save %fs, as it was saved on the
  621. * stack on entry. No need to save %es and %ds, as those are
  622. * always kernel segments while inside the kernel. Doing this
  623. * before setting the new TLS descriptors avoids the situation
  624. * where we temporarily have non-reloadable segments in %fs
  625. * and %gs. This could be an issue if the NMI handler ever
  626. * used %fs or %gs (it does not today), or if the kernel is
  627. * running inside of a hypervisor layer.
  628. */
  629. savesegment(gs, prev->gs);
  630. /*
  631. * Load the per-thread Thread-Local Storage descriptor.
  632. */
  633. load_TLS(next, cpu);
  634. /*
  635. * Restore IOPL if needed. In normal use, the flags restore
  636. * in the switch assembly will handle this. But if the kernel
  637. * is running virtualized at a non-zero CPL, the popf will
  638. * not restore flags, so it must be done in a separate step.
  639. */
  640. if (get_kernel_rpl() && unlikely(prev->iopl != next->iopl))
  641. set_iopl_mask(next->iopl);
  642. /*
  643. * Now maybe handle debug registers and/or IO bitmaps
  644. */
  645. if (unlikely(task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV ||
  646. task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
  647. __switch_to_xtra(prev_p, next_p, tss);
  648. /*
  649. * Leave lazy mode, flushing any hypercalls made here.
  650. * This must be done before restoring TLS segments so
  651. * the GDT and LDT are properly updated, and must be
  652. * done before math_state_restore, so the TS bit is up
  653. * to date.
  654. */
  655. arch_leave_lazy_cpu_mode();
  656. /* If the task has used fpu the last 5 timeslices, just do a full
  657. * restore of the math state immediately to avoid the trap; the
  658. * chances of needing FPU soon are obviously high now
  659. */
  660. if (next_p->fpu_counter > 5)
  661. math_state_restore();
  662. /*
  663. * Restore %gs if needed (which is common)
  664. */
  665. if (prev->gs | next->gs)
  666. loadsegment(gs, next->gs);
  667. x86_write_percpu(current_task, next_p);
  668. return prev_p;
  669. }
  670. asmlinkage int sys_fork(struct pt_regs regs)
  671. {
  672. return do_fork(SIGCHLD, regs.sp, &regs, 0, NULL, NULL);
  673. }
  674. asmlinkage int sys_clone(struct pt_regs regs)
  675. {
  676. unsigned long clone_flags;
  677. unsigned long newsp;
  678. int __user *parent_tidptr, *child_tidptr;
  679. clone_flags = regs.bx;
  680. newsp = regs.cx;
  681. parent_tidptr = (int __user *)regs.dx;
  682. child_tidptr = (int __user *)regs.di;
  683. if (!newsp)
  684. newsp = regs.sp;
  685. return do_fork(clone_flags, newsp, &regs, 0, parent_tidptr, child_tidptr);
  686. }
  687. /*
  688. * This is trivial, and on the face of it looks like it
  689. * could equally well be done in user mode.
  690. *
  691. * Not so, for quite unobvious reasons - register pressure.
  692. * In user mode vfork() cannot have a stack frame, and if
  693. * done by calling the "clone()" system call directly, you
  694. * do not have enough call-clobbered registers to hold all
  695. * the information you need.
  696. */
  697. asmlinkage int sys_vfork(struct pt_regs regs)
  698. {
  699. return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs.sp, &regs, 0, NULL, NULL);
  700. }
  701. /*
  702. * sys_execve() executes a new program.
  703. */
  704. asmlinkage int sys_execve(struct pt_regs regs)
  705. {
  706. int error;
  707. char * filename;
  708. filename = getname((char __user *) regs.bx);
  709. error = PTR_ERR(filename);
  710. if (IS_ERR(filename))
  711. goto out;
  712. error = do_execve(filename,
  713. (char __user * __user *) regs.cx,
  714. (char __user * __user *) regs.dx,
  715. &regs);
  716. if (error == 0) {
  717. /* Make sure we don't return using sysenter.. */
  718. set_thread_flag(TIF_IRET);
  719. }
  720. putname(filename);
  721. out:
  722. return error;
  723. }
  724. #define top_esp (THREAD_SIZE - sizeof(unsigned long))
  725. #define top_ebp (THREAD_SIZE - 2*sizeof(unsigned long))
  726. unsigned long get_wchan(struct task_struct *p)
  727. {
  728. unsigned long bp, sp, ip;
  729. unsigned long stack_page;
  730. int count = 0;
  731. if (!p || p == current || p->state == TASK_RUNNING)
  732. return 0;
  733. stack_page = (unsigned long)task_stack_page(p);
  734. sp = p->thread.sp;
  735. if (!stack_page || sp < stack_page || sp > top_esp+stack_page)
  736. return 0;
  737. /* include/asm-i386/system.h:switch_to() pushes bp last. */
  738. bp = *(unsigned long *) sp;
  739. do {
  740. if (bp < stack_page || bp > top_ebp+stack_page)
  741. return 0;
  742. ip = *(unsigned long *) (bp+4);
  743. if (!in_sched_functions(ip))
  744. return ip;
  745. bp = *(unsigned long *) bp;
  746. } while (count++ < 16);
  747. return 0;
  748. }
  749. unsigned long arch_align_stack(unsigned long sp)
  750. {
  751. if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
  752. sp -= get_random_int() % 8192;
  753. return sp & ~0xf;
  754. }
  755. unsigned long arch_randomize_brk(struct mm_struct *mm)
  756. {
  757. unsigned long range_end = mm->brk + 0x02000000;
  758. return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
  759. }