i387.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829
  1. /*
  2. * Copyright (C) 1994 Linus Torvalds
  3. *
  4. * Pentium III FXSR, SSE support
  5. * General FPU state handling cleanups
  6. * Gareth Hughes <gareth@valinux.com>, May 2000
  7. */
  8. #include <linux/module.h>
  9. #include <linux/regset.h>
  10. #include <linux/sched.h>
  11. #include <linux/slab.h>
  12. #include <asm/sigcontext.h>
  13. #include <asm/processor.h>
  14. #include <asm/math_emu.h>
  15. #include <asm/uaccess.h>
  16. #include <asm/ptrace.h>
  17. #include <asm/i387.h>
  18. #include <asm/fpu-internal.h>
  19. #include <asm/user.h>
  20. #ifdef CONFIG_X86_64
  21. # include <asm/sigcontext32.h>
  22. # include <asm/user32.h>
  23. #else
  24. # define save_i387_xstate_ia32 save_i387_xstate
  25. # define restore_i387_xstate_ia32 restore_i387_xstate
  26. # define _fpstate_ia32 _fpstate
  27. # define _xstate_ia32 _xstate
  28. # define sig_xstate_ia32_size sig_xstate_size
  29. # define fx_sw_reserved_ia32 fx_sw_reserved
  30. # define user_i387_ia32_struct user_i387_struct
  31. # define user32_fxsr_struct user_fxsr_struct
  32. #endif
  33. /*
  34. * Were we in an interrupt that interrupted kernel mode?
  35. *
  36. * We can do a kernel_fpu_begin/end() pair *ONLY* if that
  37. * pair does nothing at all: the thread must not have fpu (so
  38. * that we don't try to save the FPU state), and TS must
  39. * be set (so that the clts/stts pair does nothing that is
  40. * visible in the interrupted kernel thread).
  41. */
  42. static inline bool interrupted_kernel_fpu_idle(void)
  43. {
  44. return !__thread_has_fpu(current) &&
  45. (read_cr0() & X86_CR0_TS);
  46. }
  47. /*
  48. * Were we in user mode (or vm86 mode) when we were
  49. * interrupted?
  50. *
  51. * Doing kernel_fpu_begin/end() is ok if we are running
  52. * in an interrupt context from user mode - we'll just
  53. * save the FPU state as required.
  54. */
  55. static inline bool interrupted_user_mode(void)
  56. {
  57. struct pt_regs *regs = get_irq_regs();
  58. return regs && user_mode_vm(regs);
  59. }
  60. /*
  61. * Can we use the FPU in kernel mode with the
  62. * whole "kernel_fpu_begin/end()" sequence?
  63. *
  64. * It's always ok in process context (ie "not interrupt")
  65. * but it is sometimes ok even from an irq.
  66. */
  67. bool irq_fpu_usable(void)
  68. {
  69. return !in_interrupt() ||
  70. interrupted_user_mode() ||
  71. interrupted_kernel_fpu_idle();
  72. }
  73. EXPORT_SYMBOL(irq_fpu_usable);
  74. void kernel_fpu_begin(void)
  75. {
  76. struct task_struct *me = current;
  77. WARN_ON_ONCE(!irq_fpu_usable());
  78. preempt_disable();
  79. if (__thread_has_fpu(me)) {
  80. __save_init_fpu(me);
  81. __thread_clear_has_fpu(me);
  82. /* We do 'stts()' in kernel_fpu_end() */
  83. } else {
  84. percpu_write(fpu_owner_task, NULL);
  85. clts();
  86. }
  87. }
  88. EXPORT_SYMBOL(kernel_fpu_begin);
  89. void kernel_fpu_end(void)
  90. {
  91. stts();
  92. preempt_enable();
  93. }
  94. EXPORT_SYMBOL(kernel_fpu_end);
  95. void unlazy_fpu(struct task_struct *tsk)
  96. {
  97. preempt_disable();
  98. if (__thread_has_fpu(tsk)) {
  99. __save_init_fpu(tsk);
  100. __thread_fpu_end(tsk);
  101. } else
  102. tsk->fpu_counter = 0;
  103. preempt_enable();
  104. }
  105. EXPORT_SYMBOL(unlazy_fpu);
  106. #ifdef CONFIG_MATH_EMULATION
  107. # define HAVE_HWFP (boot_cpu_data.hard_math)
  108. #else
  109. # define HAVE_HWFP 1
  110. #endif
  111. static unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
  112. unsigned int xstate_size;
  113. EXPORT_SYMBOL_GPL(xstate_size);
  114. unsigned int sig_xstate_ia32_size = sizeof(struct _fpstate_ia32);
  115. static struct i387_fxsave_struct fx_scratch __cpuinitdata;
  116. static void __cpuinit mxcsr_feature_mask_init(void)
  117. {
  118. unsigned long mask = 0;
  119. clts();
  120. if (cpu_has_fxsr) {
  121. memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct));
  122. asm volatile("fxsave %0" : : "m" (fx_scratch));
  123. mask = fx_scratch.mxcsr_mask;
  124. if (mask == 0)
  125. mask = 0x0000ffbf;
  126. }
  127. mxcsr_feature_mask &= mask;
  128. stts();
  129. }
  130. static void __cpuinit init_thread_xstate(void)
  131. {
  132. /*
  133. * Note that xstate_size might be overwriten later during
  134. * xsave_init().
  135. */
  136. if (!HAVE_HWFP) {
  137. /*
  138. * Disable xsave as we do not support it if i387
  139. * emulation is enabled.
  140. */
  141. setup_clear_cpu_cap(X86_FEATURE_XSAVE);
  142. setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
  143. xstate_size = sizeof(struct i387_soft_struct);
  144. return;
  145. }
  146. if (cpu_has_fxsr)
  147. xstate_size = sizeof(struct i387_fxsave_struct);
  148. else
  149. xstate_size = sizeof(struct i387_fsave_struct);
  150. }
  151. /*
  152. * Called at bootup to set up the initial FPU state that is later cloned
  153. * into all processes.
  154. */
  155. void __cpuinit fpu_init(void)
  156. {
  157. unsigned long cr0;
  158. unsigned long cr4_mask = 0;
  159. if (cpu_has_fxsr)
  160. cr4_mask |= X86_CR4_OSFXSR;
  161. if (cpu_has_xmm)
  162. cr4_mask |= X86_CR4_OSXMMEXCPT;
  163. if (cr4_mask)
  164. set_in_cr4(cr4_mask);
  165. cr0 = read_cr0();
  166. cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
  167. if (!HAVE_HWFP)
  168. cr0 |= X86_CR0_EM;
  169. write_cr0(cr0);
  170. if (!smp_processor_id())
  171. init_thread_xstate();
  172. mxcsr_feature_mask_init();
  173. /* clean state in init */
  174. current_thread_info()->status = 0;
  175. clear_used_math();
  176. }
  177. void fpu_finit(struct fpu *fpu)
  178. {
  179. if (!HAVE_HWFP) {
  180. finit_soft_fpu(&fpu->state->soft);
  181. return;
  182. }
  183. if (cpu_has_fxsr) {
  184. struct i387_fxsave_struct *fx = &fpu->state->fxsave;
  185. memset(fx, 0, xstate_size);
  186. fx->cwd = 0x37f;
  187. if (cpu_has_xmm)
  188. fx->mxcsr = MXCSR_DEFAULT;
  189. } else {
  190. struct i387_fsave_struct *fp = &fpu->state->fsave;
  191. memset(fp, 0, xstate_size);
  192. fp->cwd = 0xffff037fu;
  193. fp->swd = 0xffff0000u;
  194. fp->twd = 0xffffffffu;
  195. fp->fos = 0xffff0000u;
  196. }
  197. }
  198. EXPORT_SYMBOL_GPL(fpu_finit);
  199. /*
  200. * The _current_ task is using the FPU for the first time
  201. * so initialize it and set the mxcsr to its default
  202. * value at reset if we support XMM instructions and then
  203. * remember the current task has used the FPU.
  204. */
  205. int init_fpu(struct task_struct *tsk)
  206. {
  207. int ret;
  208. if (tsk_used_math(tsk)) {
  209. if (HAVE_HWFP && tsk == current)
  210. unlazy_fpu(tsk);
  211. return 0;
  212. }
  213. /*
  214. * Memory allocation at the first usage of the FPU and other state.
  215. */
  216. ret = fpu_alloc(&tsk->thread.fpu);
  217. if (ret)
  218. return ret;
  219. fpu_finit(&tsk->thread.fpu);
  220. set_stopped_child_used_math(tsk);
  221. return 0;
  222. }
  223. EXPORT_SYMBOL_GPL(init_fpu);
  224. /*
  225. * The xstateregs_active() routine is the same as the fpregs_active() routine,
  226. * as the "regset->n" for the xstate regset will be updated based on the feature
  227. * capabilites supported by the xsave.
  228. */
  229. int fpregs_active(struct task_struct *target, const struct user_regset *regset)
  230. {
  231. return tsk_used_math(target) ? regset->n : 0;
  232. }
  233. int xfpregs_active(struct task_struct *target, const struct user_regset *regset)
  234. {
  235. return (cpu_has_fxsr && tsk_used_math(target)) ? regset->n : 0;
  236. }
  237. int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
  238. unsigned int pos, unsigned int count,
  239. void *kbuf, void __user *ubuf)
  240. {
  241. int ret;
  242. if (!cpu_has_fxsr)
  243. return -ENODEV;
  244. ret = init_fpu(target);
  245. if (ret)
  246. return ret;
  247. sanitize_i387_state(target);
  248. return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
  249. &target->thread.fpu.state->fxsave, 0, -1);
  250. }
  251. int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
  252. unsigned int pos, unsigned int count,
  253. const void *kbuf, const void __user *ubuf)
  254. {
  255. int ret;
  256. if (!cpu_has_fxsr)
  257. return -ENODEV;
  258. ret = init_fpu(target);
  259. if (ret)
  260. return ret;
  261. sanitize_i387_state(target);
  262. ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
  263. &target->thread.fpu.state->fxsave, 0, -1);
  264. /*
  265. * mxcsr reserved bits must be masked to zero for security reasons.
  266. */
  267. target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
  268. /*
  269. * update the header bits in the xsave header, indicating the
  270. * presence of FP and SSE state.
  271. */
  272. if (cpu_has_xsave)
  273. target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
  274. return ret;
  275. }
  276. int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
  277. unsigned int pos, unsigned int count,
  278. void *kbuf, void __user *ubuf)
  279. {
  280. int ret;
  281. if (!cpu_has_xsave)
  282. return -ENODEV;
  283. ret = init_fpu(target);
  284. if (ret)
  285. return ret;
  286. /*
  287. * Copy the 48bytes defined by the software first into the xstate
  288. * memory layout in the thread struct, so that we can copy the entire
  289. * xstateregs to the user using one user_regset_copyout().
  290. */
  291. memcpy(&target->thread.fpu.state->fxsave.sw_reserved,
  292. xstate_fx_sw_bytes, sizeof(xstate_fx_sw_bytes));
  293. /*
  294. * Copy the xstate memory layout.
  295. */
  296. ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
  297. &target->thread.fpu.state->xsave, 0, -1);
  298. return ret;
  299. }
  300. int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
  301. unsigned int pos, unsigned int count,
  302. const void *kbuf, const void __user *ubuf)
  303. {
  304. int ret;
  305. struct xsave_hdr_struct *xsave_hdr;
  306. if (!cpu_has_xsave)
  307. return -ENODEV;
  308. ret = init_fpu(target);
  309. if (ret)
  310. return ret;
  311. ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
  312. &target->thread.fpu.state->xsave, 0, -1);
  313. /*
  314. * mxcsr reserved bits must be masked to zero for security reasons.
  315. */
  316. target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
  317. xsave_hdr = &target->thread.fpu.state->xsave.xsave_hdr;
  318. xsave_hdr->xstate_bv &= pcntxt_mask;
  319. /*
  320. * These bits must be zero.
  321. */
  322. xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
  323. return ret;
  324. }
  325. #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
  326. /*
  327. * FPU tag word conversions.
  328. */
  329. static inline unsigned short twd_i387_to_fxsr(unsigned short twd)
  330. {
  331. unsigned int tmp; /* to avoid 16 bit prefixes in the code */
  332. /* Transform each pair of bits into 01 (valid) or 00 (empty) */
  333. tmp = ~twd;
  334. tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
  335. /* and move the valid bits to the lower byte. */
  336. tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
  337. tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
  338. tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
  339. return tmp;
  340. }
  341. #define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16)
  342. #define FP_EXP_TAG_VALID 0
  343. #define FP_EXP_TAG_ZERO 1
  344. #define FP_EXP_TAG_SPECIAL 2
  345. #define FP_EXP_TAG_EMPTY 3
  346. static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
  347. {
  348. struct _fpxreg *st;
  349. u32 tos = (fxsave->swd >> 11) & 7;
  350. u32 twd = (unsigned long) fxsave->twd;
  351. u32 tag;
  352. u32 ret = 0xffff0000u;
  353. int i;
  354. for (i = 0; i < 8; i++, twd >>= 1) {
  355. if (twd & 0x1) {
  356. st = FPREG_ADDR(fxsave, (i - tos) & 7);
  357. switch (st->exponent & 0x7fff) {
  358. case 0x7fff:
  359. tag = FP_EXP_TAG_SPECIAL;
  360. break;
  361. case 0x0000:
  362. if (!st->significand[0] &&
  363. !st->significand[1] &&
  364. !st->significand[2] &&
  365. !st->significand[3])
  366. tag = FP_EXP_TAG_ZERO;
  367. else
  368. tag = FP_EXP_TAG_SPECIAL;
  369. break;
  370. default:
  371. if (st->significand[3] & 0x8000)
  372. tag = FP_EXP_TAG_VALID;
  373. else
  374. tag = FP_EXP_TAG_SPECIAL;
  375. break;
  376. }
  377. } else {
  378. tag = FP_EXP_TAG_EMPTY;
  379. }
  380. ret |= tag << (2 * i);
  381. }
  382. return ret;
  383. }
  384. /*
  385. * FXSR floating point environment conversions.
  386. */
  387. static void
  388. convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
  389. {
  390. struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
  391. struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
  392. struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
  393. int i;
  394. env->cwd = fxsave->cwd | 0xffff0000u;
  395. env->swd = fxsave->swd | 0xffff0000u;
  396. env->twd = twd_fxsr_to_i387(fxsave);
  397. #ifdef CONFIG_X86_64
  398. env->fip = fxsave->rip;
  399. env->foo = fxsave->rdp;
  400. /*
  401. * should be actually ds/cs at fpu exception time, but
  402. * that information is not available in 64bit mode.
  403. */
  404. env->fcs = task_pt_regs(tsk)->cs;
  405. if (tsk == current) {
  406. savesegment(ds, env->fos);
  407. } else {
  408. env->fos = tsk->thread.ds;
  409. }
  410. env->fos |= 0xffff0000;
  411. #else
  412. env->fip = fxsave->fip;
  413. env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);
  414. env->foo = fxsave->foo;
  415. env->fos = fxsave->fos;
  416. #endif
  417. for (i = 0; i < 8; ++i)
  418. memcpy(&to[i], &from[i], sizeof(to[0]));
  419. }
  420. static void convert_to_fxsr(struct task_struct *tsk,
  421. const struct user_i387_ia32_struct *env)
  422. {
  423. struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
  424. struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
  425. struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
  426. int i;
  427. fxsave->cwd = env->cwd;
  428. fxsave->swd = env->swd;
  429. fxsave->twd = twd_i387_to_fxsr(env->twd);
  430. fxsave->fop = (u16) ((u32) env->fcs >> 16);
  431. #ifdef CONFIG_X86_64
  432. fxsave->rip = env->fip;
  433. fxsave->rdp = env->foo;
  434. /* cs and ds ignored */
  435. #else
  436. fxsave->fip = env->fip;
  437. fxsave->fcs = (env->fcs & 0xffff);
  438. fxsave->foo = env->foo;
  439. fxsave->fos = env->fos;
  440. #endif
  441. for (i = 0; i < 8; ++i)
  442. memcpy(&to[i], &from[i], sizeof(from[0]));
  443. }
  444. int fpregs_get(struct task_struct *target, const struct user_regset *regset,
  445. unsigned int pos, unsigned int count,
  446. void *kbuf, void __user *ubuf)
  447. {
  448. struct user_i387_ia32_struct env;
  449. int ret;
  450. ret = init_fpu(target);
  451. if (ret)
  452. return ret;
  453. if (!HAVE_HWFP)
  454. return fpregs_soft_get(target, regset, pos, count, kbuf, ubuf);
  455. if (!cpu_has_fxsr) {
  456. return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
  457. &target->thread.fpu.state->fsave, 0,
  458. -1);
  459. }
  460. sanitize_i387_state(target);
  461. if (kbuf && pos == 0 && count == sizeof(env)) {
  462. convert_from_fxsr(kbuf, target);
  463. return 0;
  464. }
  465. convert_from_fxsr(&env, target);
  466. return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
  467. }
  468. int fpregs_set(struct task_struct *target, const struct user_regset *regset,
  469. unsigned int pos, unsigned int count,
  470. const void *kbuf, const void __user *ubuf)
  471. {
  472. struct user_i387_ia32_struct env;
  473. int ret;
  474. ret = init_fpu(target);
  475. if (ret)
  476. return ret;
  477. sanitize_i387_state(target);
  478. if (!HAVE_HWFP)
  479. return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
  480. if (!cpu_has_fxsr) {
  481. return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
  482. &target->thread.fpu.state->fsave, 0, -1);
  483. }
  484. if (pos > 0 || count < sizeof(env))
  485. convert_from_fxsr(&env, target);
  486. ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
  487. if (!ret)
  488. convert_to_fxsr(target, &env);
  489. /*
  490. * update the header bit in the xsave header, indicating the
  491. * presence of FP.
  492. */
  493. if (cpu_has_xsave)
  494. target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FP;
  495. return ret;
  496. }
  497. /*
  498. * Signal frame handlers.
  499. */
  500. static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf)
  501. {
  502. struct task_struct *tsk = current;
  503. struct i387_fsave_struct *fp = &tsk->thread.fpu.state->fsave;
  504. fp->status = fp->swd;
  505. if (__copy_to_user(buf, fp, sizeof(struct i387_fsave_struct)))
  506. return -1;
  507. return 1;
  508. }
  509. static int save_i387_fxsave(struct _fpstate_ia32 __user *buf)
  510. {
  511. struct task_struct *tsk = current;
  512. struct i387_fxsave_struct *fx = &tsk->thread.fpu.state->fxsave;
  513. struct user_i387_ia32_struct env;
  514. int err = 0;
  515. convert_from_fxsr(&env, tsk);
  516. if (__copy_to_user(buf, &env, sizeof(env)))
  517. return -1;
  518. err |= __put_user(fx->swd, &buf->status);
  519. err |= __put_user(X86_FXSR_MAGIC, &buf->magic);
  520. if (err)
  521. return -1;
  522. if (__copy_to_user(&buf->_fxsr_env[0], fx, xstate_size))
  523. return -1;
  524. return 1;
  525. }
  526. static int save_i387_xsave(void __user *buf)
  527. {
  528. struct task_struct *tsk = current;
  529. struct _fpstate_ia32 __user *fx = buf;
  530. int err = 0;
  531. sanitize_i387_state(tsk);
  532. /*
  533. * For legacy compatible, we always set FP/SSE bits in the bit
  534. * vector while saving the state to the user context.
  535. * This will enable us capturing any changes(during sigreturn) to
  536. * the FP/SSE bits by the legacy applications which don't touch
  537. * xstate_bv in the xsave header.
  538. *
  539. * xsave aware applications can change the xstate_bv in the xsave
  540. * header as well as change any contents in the memory layout.
  541. * xrestore as part of sigreturn will capture all the changes.
  542. */
  543. tsk->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
  544. if (save_i387_fxsave(fx) < 0)
  545. return -1;
  546. err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved_ia32,
  547. sizeof(struct _fpx_sw_bytes));
  548. err |= __put_user(FP_XSTATE_MAGIC2,
  549. (__u32 __user *) (buf + sig_xstate_ia32_size
  550. - FP_XSTATE_MAGIC2_SIZE));
  551. if (err)
  552. return -1;
  553. return 1;
  554. }
  555. int save_i387_xstate_ia32(void __user *buf)
  556. {
  557. struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
  558. struct task_struct *tsk = current;
  559. if (!used_math())
  560. return 0;
  561. if (!access_ok(VERIFY_WRITE, buf, sig_xstate_ia32_size))
  562. return -EACCES;
  563. /*
  564. * This will cause a "finit" to be triggered by the next
  565. * attempted FPU operation by the 'current' process.
  566. */
  567. clear_used_math();
  568. if (!HAVE_HWFP) {
  569. return fpregs_soft_get(current, NULL,
  570. 0, sizeof(struct user_i387_ia32_struct),
  571. NULL, fp) ? -1 : 1;
  572. }
  573. unlazy_fpu(tsk);
  574. if (cpu_has_xsave)
  575. return save_i387_xsave(fp);
  576. if (cpu_has_fxsr)
  577. return save_i387_fxsave(fp);
  578. else
  579. return save_i387_fsave(fp);
  580. }
  581. static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf)
  582. {
  583. struct task_struct *tsk = current;
  584. return __copy_from_user(&tsk->thread.fpu.state->fsave, buf,
  585. sizeof(struct i387_fsave_struct));
  586. }
  587. static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf,
  588. unsigned int size)
  589. {
  590. struct task_struct *tsk = current;
  591. struct user_i387_ia32_struct env;
  592. int err;
  593. err = __copy_from_user(&tsk->thread.fpu.state->fxsave, &buf->_fxsr_env[0],
  594. size);
  595. /* mxcsr reserved bits must be masked to zero for security reasons */
  596. tsk->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
  597. if (err || __copy_from_user(&env, buf, sizeof(env)))
  598. return 1;
  599. convert_to_fxsr(tsk, &env);
  600. return 0;
  601. }
  602. static int restore_i387_xsave(void __user *buf)
  603. {
  604. struct _fpx_sw_bytes fx_sw_user;
  605. struct _fpstate_ia32 __user *fx_user =
  606. ((struct _fpstate_ia32 __user *) buf);
  607. struct i387_fxsave_struct __user *fx =
  608. (struct i387_fxsave_struct __user *) &fx_user->_fxsr_env[0];
  609. struct xsave_hdr_struct *xsave_hdr =
  610. &current->thread.fpu.state->xsave.xsave_hdr;
  611. u64 mask;
  612. int err;
  613. if (check_for_xstate(fx, buf, &fx_sw_user))
  614. goto fx_only;
  615. mask = fx_sw_user.xstate_bv;
  616. err = restore_i387_fxsave(buf, fx_sw_user.xstate_size);
  617. xsave_hdr->xstate_bv &= pcntxt_mask;
  618. /*
  619. * These bits must be zero.
  620. */
  621. xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
  622. /*
  623. * Init the state that is not present in the memory layout
  624. * and enabled by the OS.
  625. */
  626. mask = ~(pcntxt_mask & ~mask);
  627. xsave_hdr->xstate_bv &= mask;
  628. return err;
  629. fx_only:
  630. /*
  631. * Couldn't find the extended state information in the memory
  632. * layout. Restore the FP/SSE and init the other extended state
  633. * enabled by the OS.
  634. */
  635. xsave_hdr->xstate_bv = XSTATE_FPSSE;
  636. return restore_i387_fxsave(buf, sizeof(struct i387_fxsave_struct));
  637. }
  638. int restore_i387_xstate_ia32(void __user *buf)
  639. {
  640. int err;
  641. struct task_struct *tsk = current;
  642. struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
  643. if (HAVE_HWFP)
  644. clear_fpu(tsk);
  645. if (!buf) {
  646. if (used_math()) {
  647. clear_fpu(tsk);
  648. clear_used_math();
  649. }
  650. return 0;
  651. } else
  652. if (!access_ok(VERIFY_READ, buf, sig_xstate_ia32_size))
  653. return -EACCES;
  654. if (!used_math()) {
  655. err = init_fpu(tsk);
  656. if (err)
  657. return err;
  658. }
  659. if (HAVE_HWFP) {
  660. if (cpu_has_xsave)
  661. err = restore_i387_xsave(buf);
  662. else if (cpu_has_fxsr)
  663. err = restore_i387_fxsave(fp, sizeof(struct
  664. i387_fxsave_struct));
  665. else
  666. err = restore_i387_fsave(fp);
  667. } else {
  668. err = fpregs_soft_set(current, NULL,
  669. 0, sizeof(struct user_i387_ia32_struct),
  670. NULL, fp) != 0;
  671. }
  672. set_used_math();
  673. return err;
  674. }
  675. /*
  676. * FPU state for core dumps.
  677. * This is only used for a.out dumps now.
  678. * It is declared generically using elf_fpregset_t (which is
  679. * struct user_i387_struct) but is in fact only used for 32-bit
  680. * dumps, so on 64-bit it is really struct user_i387_ia32_struct.
  681. */
  682. int dump_fpu(struct pt_regs *regs, struct user_i387_struct *fpu)
  683. {
  684. struct task_struct *tsk = current;
  685. int fpvalid;
  686. fpvalid = !!used_math();
  687. if (fpvalid)
  688. fpvalid = !fpregs_get(tsk, NULL,
  689. 0, sizeof(struct user_i387_ia32_struct),
  690. fpu, NULL);
  691. return fpvalid;
  692. }
  693. EXPORT_SYMBOL(dump_fpu);
  694. #endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */