paravirt.h 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092
  1. #ifndef _ASM_X86_PARAVIRT_H
  2. #define _ASM_X86_PARAVIRT_H
  3. /* Various instructions on x86 need to be replaced for
  4. * para-virtualization: those hooks are defined here. */
  5. #ifdef CONFIG_PARAVIRT
  6. #include <asm/pgtable_types.h>
  7. #include <asm/asm.h>
  8. #include <asm/paravirt_types.h>
  9. #ifndef __ASSEMBLY__
  10. #include <linux/types.h>
  11. #include <linux/cpumask.h>
  12. static inline int paravirt_enabled(void)
  13. {
  14. return pv_info.paravirt_enabled;
  15. }
  16. static inline void load_sp0(struct tss_struct *tss,
  17. struct thread_struct *thread)
  18. {
  19. PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
  20. }
  21. #define ARCH_SETUP pv_init_ops.arch_setup();
  22. static inline unsigned long get_wallclock(void)
  23. {
  24. return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock);
  25. }
  26. static inline int set_wallclock(unsigned long nowtime)
  27. {
  28. return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime);
  29. }
  30. static inline void (*choose_time_init(void))(void)
  31. {
  32. return pv_time_ops.time_init;
  33. }
  34. /* The paravirtualized CPUID instruction. */
  35. static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
  36. unsigned int *ecx, unsigned int *edx)
  37. {
  38. PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
  39. }
  40. /*
  41. * These special macros can be used to get or set a debugging register
  42. */
  43. static inline unsigned long paravirt_get_debugreg(int reg)
  44. {
  45. return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
  46. }
  47. #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
  48. static inline void set_debugreg(unsigned long val, int reg)
  49. {
  50. PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
  51. }
  52. static inline void clts(void)
  53. {
  54. PVOP_VCALL0(pv_cpu_ops.clts);
  55. }
  56. static inline unsigned long read_cr0(void)
  57. {
  58. return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
  59. }
  60. static inline void write_cr0(unsigned long x)
  61. {
  62. PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
  63. }
  64. static inline unsigned long read_cr2(void)
  65. {
  66. return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
  67. }
  68. static inline void write_cr2(unsigned long x)
  69. {
  70. PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
  71. }
  72. static inline unsigned long read_cr3(void)
  73. {
  74. return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
  75. }
  76. static inline void write_cr3(unsigned long x)
  77. {
  78. PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
  79. }
  80. static inline unsigned long read_cr4(void)
  81. {
  82. return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
  83. }
  84. static inline unsigned long read_cr4_safe(void)
  85. {
  86. return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
  87. }
  88. static inline void write_cr4(unsigned long x)
  89. {
  90. PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
  91. }
  92. #ifdef CONFIG_X86_64
  93. static inline unsigned long read_cr8(void)
  94. {
  95. return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
  96. }
  97. static inline void write_cr8(unsigned long x)
  98. {
  99. PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
  100. }
  101. #endif
  102. static inline void raw_safe_halt(void)
  103. {
  104. PVOP_VCALL0(pv_irq_ops.safe_halt);
  105. }
  106. static inline void halt(void)
  107. {
  108. PVOP_VCALL0(pv_irq_ops.safe_halt);
  109. }
  110. static inline void wbinvd(void)
  111. {
  112. PVOP_VCALL0(pv_cpu_ops.wbinvd);
  113. }
  114. #define get_kernel_rpl() (pv_info.kernel_rpl)
  115. static inline u64 paravirt_read_msr(unsigned msr, int *err)
  116. {
  117. return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
  118. }
  119. static inline int paravirt_rdmsr_regs(u32 *regs)
  120. {
  121. return PVOP_CALL1(int, pv_cpu_ops.rdmsr_regs, regs);
  122. }
  123. static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
  124. {
  125. return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
  126. }
  127. static inline int paravirt_wrmsr_regs(u32 *regs)
  128. {
  129. return PVOP_CALL1(int, pv_cpu_ops.wrmsr_regs, regs);
  130. }
  131. /* These should all do BUG_ON(_err), but our headers are too tangled. */
  132. #define rdmsr(msr, val1, val2) \
  133. do { \
  134. int _err; \
  135. u64 _l = paravirt_read_msr(msr, &_err); \
  136. val1 = (u32)_l; \
  137. val2 = _l >> 32; \
  138. } while (0)
  139. #define wrmsr(msr, val1, val2) \
  140. do { \
  141. paravirt_write_msr(msr, val1, val2); \
  142. } while (0)
  143. #define rdmsrl(msr, val) \
  144. do { \
  145. int _err; \
  146. val = paravirt_read_msr(msr, &_err); \
  147. } while (0)
  148. #define wrmsrl(msr, val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32)
  149. #define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b)
  150. /* rdmsr with exception handling */
  151. #define rdmsr_safe(msr, a, b) \
  152. ({ \
  153. int _err; \
  154. u64 _l = paravirt_read_msr(msr, &_err); \
  155. (*a) = (u32)_l; \
  156. (*b) = _l >> 32; \
  157. _err; \
  158. })
  159. #define rdmsr_safe_regs(regs) paravirt_rdmsr_regs(regs)
  160. #define wrmsr_safe_regs(regs) paravirt_wrmsr_regs(regs)
  161. static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
  162. {
  163. int err;
  164. *p = paravirt_read_msr(msr, &err);
  165. return err;
  166. }
  167. static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
  168. {
  169. u32 gprs[8] = { 0 };
  170. int err;
  171. gprs[1] = msr;
  172. gprs[7] = 0x9c5a203a;
  173. err = paravirt_rdmsr_regs(gprs);
  174. *p = gprs[0] | ((u64)gprs[2] << 32);
  175. return err;
  176. }
  177. static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
  178. {
  179. u32 gprs[8] = { 0 };
  180. gprs[0] = (u32)val;
  181. gprs[1] = msr;
  182. gprs[2] = val >> 32;
  183. gprs[7] = 0x9c5a203a;
  184. return paravirt_wrmsr_regs(gprs);
  185. }
  186. static inline u64 paravirt_read_tsc(void)
  187. {
  188. return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
  189. }
  190. #define rdtscl(low) \
  191. do { \
  192. u64 _l = paravirt_read_tsc(); \
  193. low = (int)_l; \
  194. } while (0)
  195. #define rdtscll(val) (val = paravirt_read_tsc())
  196. static inline unsigned long long paravirt_sched_clock(void)
  197. {
  198. return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
  199. }
  200. #define calibrate_tsc() (pv_time_ops.get_tsc_khz())
  201. static inline unsigned long long paravirt_read_pmc(int counter)
  202. {
  203. return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
  204. }
  205. #define rdpmc(counter, low, high) \
  206. do { \
  207. u64 _l = paravirt_read_pmc(counter); \
  208. low = (u32)_l; \
  209. high = _l >> 32; \
  210. } while (0)
  211. static inline unsigned long long paravirt_rdtscp(unsigned int *aux)
  212. {
  213. return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux);
  214. }
  215. #define rdtscp(low, high, aux) \
  216. do { \
  217. int __aux; \
  218. unsigned long __val = paravirt_rdtscp(&__aux); \
  219. (low) = (u32)__val; \
  220. (high) = (u32)(__val >> 32); \
  221. (aux) = __aux; \
  222. } while (0)
  223. #define rdtscpll(val, aux) \
  224. do { \
  225. unsigned long __aux; \
  226. val = paravirt_rdtscp(&__aux); \
  227. (aux) = __aux; \
  228. } while (0)
  229. static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
  230. {
  231. PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
  232. }
  233. static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
  234. {
  235. PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
  236. }
  237. static inline void load_TR_desc(void)
  238. {
  239. PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
  240. }
  241. static inline void load_gdt(const struct desc_ptr *dtr)
  242. {
  243. PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
  244. }
  245. static inline void load_idt(const struct desc_ptr *dtr)
  246. {
  247. PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
  248. }
  249. static inline void set_ldt(const void *addr, unsigned entries)
  250. {
  251. PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
  252. }
  253. static inline void store_gdt(struct desc_ptr *dtr)
  254. {
  255. PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr);
  256. }
  257. static inline void store_idt(struct desc_ptr *dtr)
  258. {
  259. PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
  260. }
  261. static inline unsigned long paravirt_store_tr(void)
  262. {
  263. return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
  264. }
  265. #define store_tr(tr) ((tr) = paravirt_store_tr())
  266. static inline void load_TLS(struct thread_struct *t, unsigned cpu)
  267. {
  268. PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
  269. }
  270. #ifdef CONFIG_X86_64
  271. static inline void load_gs_index(unsigned int gs)
  272. {
  273. PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
  274. }
  275. #endif
  276. static inline void write_ldt_entry(struct desc_struct *dt, int entry,
  277. const void *desc)
  278. {
  279. PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
  280. }
  281. static inline void write_gdt_entry(struct desc_struct *dt, int entry,
  282. void *desc, int type)
  283. {
  284. PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
  285. }
  286. static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
  287. {
  288. PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
  289. }
  290. static inline void set_iopl_mask(unsigned mask)
  291. {
  292. PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
  293. }
  294. /* The paravirtualized I/O functions */
  295. static inline void slow_down_io(void)
  296. {
  297. pv_cpu_ops.io_delay();
  298. #ifdef REALLY_SLOW_IO
  299. pv_cpu_ops.io_delay();
  300. pv_cpu_ops.io_delay();
  301. pv_cpu_ops.io_delay();
  302. #endif
  303. }
  304. #ifdef CONFIG_X86_LOCAL_APIC
  305. static inline void setup_boot_clock(void)
  306. {
  307. PVOP_VCALL0(pv_apic_ops.setup_boot_clock);
  308. }
  309. static inline void setup_secondary_clock(void)
  310. {
  311. PVOP_VCALL0(pv_apic_ops.setup_secondary_clock);
  312. }
  313. #endif
  314. static inline void paravirt_post_allocator_init(void)
  315. {
  316. if (pv_init_ops.post_allocator_init)
  317. (*pv_init_ops.post_allocator_init)();
  318. }
  319. static inline void paravirt_pagetable_setup_start(pgd_t *base)
  320. {
  321. (*pv_mmu_ops.pagetable_setup_start)(base);
  322. }
  323. static inline void paravirt_pagetable_setup_done(pgd_t *base)
  324. {
  325. (*pv_mmu_ops.pagetable_setup_done)(base);
  326. }
  327. #ifdef CONFIG_SMP
  328. static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
  329. unsigned long start_esp)
  330. {
  331. PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
  332. phys_apicid, start_eip, start_esp);
  333. }
  334. #endif
  335. static inline void paravirt_activate_mm(struct mm_struct *prev,
  336. struct mm_struct *next)
  337. {
  338. PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
  339. }
  340. static inline void arch_dup_mmap(struct mm_struct *oldmm,
  341. struct mm_struct *mm)
  342. {
  343. PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
  344. }
  345. static inline void arch_exit_mmap(struct mm_struct *mm)
  346. {
  347. PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
  348. }
  349. static inline void __flush_tlb(void)
  350. {
  351. PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
  352. }
  353. static inline void __flush_tlb_global(void)
  354. {
  355. PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
  356. }
  357. static inline void __flush_tlb_single(unsigned long addr)
  358. {
  359. PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
  360. }
  361. static inline void flush_tlb_others(const struct cpumask *cpumask,
  362. struct mm_struct *mm,
  363. unsigned long va)
  364. {
  365. PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va);
  366. }
  367. static inline int paravirt_pgd_alloc(struct mm_struct *mm)
  368. {
  369. return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
  370. }
  371. static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
  372. {
  373. PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
  374. }
  375. static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
  376. {
  377. PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
  378. }
  379. static inline void paravirt_release_pte(unsigned long pfn)
  380. {
  381. PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
  382. }
  383. static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
  384. {
  385. PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
  386. }
  387. static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
  388. unsigned long start, unsigned long count)
  389. {
  390. PVOP_VCALL4(pv_mmu_ops.alloc_pmd_clone, pfn, clonepfn, start, count);
  391. }
  392. static inline void paravirt_release_pmd(unsigned long pfn)
  393. {
  394. PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
  395. }
  396. static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
  397. {
  398. PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
  399. }
  400. static inline void paravirt_release_pud(unsigned long pfn)
  401. {
  402. PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
  403. }
  404. #ifdef CONFIG_HIGHPTE
  405. static inline void *kmap_atomic_pte(struct page *page, enum km_type type)
  406. {
  407. unsigned long ret;
  408. ret = PVOP_CALL2(unsigned long, pv_mmu_ops.kmap_atomic_pte, page, type);
  409. return (void *)ret;
  410. }
  411. #endif
  412. static inline void pte_update(struct mm_struct *mm, unsigned long addr,
  413. pte_t *ptep)
  414. {
  415. PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
  416. }
  417. static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
  418. pte_t *ptep)
  419. {
  420. PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
  421. }
  422. static inline pte_t __pte(pteval_t val)
  423. {
  424. pteval_t ret;
  425. if (sizeof(pteval_t) > sizeof(long))
  426. ret = PVOP_CALLEE2(pteval_t,
  427. pv_mmu_ops.make_pte,
  428. val, (u64)val >> 32);
  429. else
  430. ret = PVOP_CALLEE1(pteval_t,
  431. pv_mmu_ops.make_pte,
  432. val);
  433. return (pte_t) { .pte = ret };
  434. }
  435. static inline pteval_t pte_val(pte_t pte)
  436. {
  437. pteval_t ret;
  438. if (sizeof(pteval_t) > sizeof(long))
  439. ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
  440. pte.pte, (u64)pte.pte >> 32);
  441. else
  442. ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
  443. pte.pte);
  444. return ret;
  445. }
  446. static inline pgd_t __pgd(pgdval_t val)
  447. {
  448. pgdval_t ret;
  449. if (sizeof(pgdval_t) > sizeof(long))
  450. ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
  451. val, (u64)val >> 32);
  452. else
  453. ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
  454. val);
  455. return (pgd_t) { ret };
  456. }
  457. static inline pgdval_t pgd_val(pgd_t pgd)
  458. {
  459. pgdval_t ret;
  460. if (sizeof(pgdval_t) > sizeof(long))
  461. ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
  462. pgd.pgd, (u64)pgd.pgd >> 32);
  463. else
  464. ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
  465. pgd.pgd);
  466. return ret;
  467. }
  468. #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
  469. static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
  470. pte_t *ptep)
  471. {
  472. pteval_t ret;
  473. ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
  474. mm, addr, ptep);
  475. return (pte_t) { .pte = ret };
  476. }
  477. static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
  478. pte_t *ptep, pte_t pte)
  479. {
  480. if (sizeof(pteval_t) > sizeof(long))
  481. /* 5 arg words */
  482. pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
  483. else
  484. PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
  485. mm, addr, ptep, pte.pte);
  486. }
  487. static inline void set_pte(pte_t *ptep, pte_t pte)
  488. {
  489. if (sizeof(pteval_t) > sizeof(long))
  490. PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
  491. pte.pte, (u64)pte.pte >> 32);
  492. else
  493. PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
  494. pte.pte);
  495. }
  496. static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
  497. pte_t *ptep, pte_t pte)
  498. {
  499. if (sizeof(pteval_t) > sizeof(long))
  500. /* 5 arg words */
  501. pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
  502. else
  503. PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
  504. }
  505. static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
  506. {
  507. pmdval_t val = native_pmd_val(pmd);
  508. if (sizeof(pmdval_t) > sizeof(long))
  509. PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
  510. else
  511. PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
  512. }
  513. #if PAGETABLE_LEVELS >= 3
  514. static inline pmd_t __pmd(pmdval_t val)
  515. {
  516. pmdval_t ret;
  517. if (sizeof(pmdval_t) > sizeof(long))
  518. ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
  519. val, (u64)val >> 32);
  520. else
  521. ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
  522. val);
  523. return (pmd_t) { ret };
  524. }
  525. static inline pmdval_t pmd_val(pmd_t pmd)
  526. {
  527. pmdval_t ret;
  528. if (sizeof(pmdval_t) > sizeof(long))
  529. ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
  530. pmd.pmd, (u64)pmd.pmd >> 32);
  531. else
  532. ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
  533. pmd.pmd);
  534. return ret;
  535. }
  536. static inline void set_pud(pud_t *pudp, pud_t pud)
  537. {
  538. pudval_t val = native_pud_val(pud);
  539. if (sizeof(pudval_t) > sizeof(long))
  540. PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
  541. val, (u64)val >> 32);
  542. else
  543. PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
  544. val);
  545. }
  546. #if PAGETABLE_LEVELS == 4
  547. static inline pud_t __pud(pudval_t val)
  548. {
  549. pudval_t ret;
  550. if (sizeof(pudval_t) > sizeof(long))
  551. ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
  552. val, (u64)val >> 32);
  553. else
  554. ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
  555. val);
  556. return (pud_t) { ret };
  557. }
  558. static inline pudval_t pud_val(pud_t pud)
  559. {
  560. pudval_t ret;
  561. if (sizeof(pudval_t) > sizeof(long))
  562. ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
  563. pud.pud, (u64)pud.pud >> 32);
  564. else
  565. ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
  566. pud.pud);
  567. return ret;
  568. }
  569. static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
  570. {
  571. pgdval_t val = native_pgd_val(pgd);
  572. if (sizeof(pgdval_t) > sizeof(long))
  573. PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
  574. val, (u64)val >> 32);
  575. else
  576. PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
  577. val);
  578. }
  579. static inline void pgd_clear(pgd_t *pgdp)
  580. {
  581. set_pgd(pgdp, __pgd(0));
  582. }
  583. static inline void pud_clear(pud_t *pudp)
  584. {
  585. set_pud(pudp, __pud(0));
  586. }
  587. #endif /* PAGETABLE_LEVELS == 4 */
  588. #endif /* PAGETABLE_LEVELS >= 3 */
  589. #ifdef CONFIG_X86_PAE
  590. /* Special-case pte-setting operations for PAE, which can't update a
  591. 64-bit pte atomically */
  592. static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
  593. {
  594. PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
  595. pte.pte, pte.pte >> 32);
  596. }
  597. static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
  598. pte_t *ptep)
  599. {
  600. PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
  601. }
  602. static inline void pmd_clear(pmd_t *pmdp)
  603. {
  604. PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
  605. }
  606. #else /* !CONFIG_X86_PAE */
  607. static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
  608. {
  609. set_pte(ptep, pte);
  610. }
  611. static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
  612. pte_t *ptep)
  613. {
  614. set_pte_at(mm, addr, ptep, __pte(0));
  615. }
  616. static inline void pmd_clear(pmd_t *pmdp)
  617. {
  618. set_pmd(pmdp, __pmd(0));
  619. }
  620. #endif /* CONFIG_X86_PAE */
  621. #define __HAVE_ARCH_START_CONTEXT_SWITCH
  622. static inline void arch_start_context_switch(struct task_struct *prev)
  623. {
  624. PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
  625. }
  626. static inline void arch_end_context_switch(struct task_struct *next)
  627. {
  628. PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
  629. }
  630. #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
  631. static inline void arch_enter_lazy_mmu_mode(void)
  632. {
  633. PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
  634. }
  635. static inline void arch_leave_lazy_mmu_mode(void)
  636. {
  637. PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
  638. }
  639. void arch_flush_lazy_mmu_mode(void);
  640. static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
  641. phys_addr_t phys, pgprot_t flags)
  642. {
  643. pv_mmu_ops.set_fixmap(idx, phys, flags);
  644. }
  645. #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
  646. static inline int __raw_spin_is_locked(struct raw_spinlock *lock)
  647. {
  648. return PVOP_CALL1(int, pv_lock_ops.spin_is_locked, lock);
  649. }
  650. static inline int __raw_spin_is_contended(struct raw_spinlock *lock)
  651. {
  652. return PVOP_CALL1(int, pv_lock_ops.spin_is_contended, lock);
  653. }
  654. #define __raw_spin_is_contended __raw_spin_is_contended
  655. static __always_inline void __raw_spin_lock(struct raw_spinlock *lock)
  656. {
  657. PVOP_VCALL1(pv_lock_ops.spin_lock, lock);
  658. }
  659. static __always_inline void __raw_spin_lock_flags(struct raw_spinlock *lock,
  660. unsigned long flags)
  661. {
  662. PVOP_VCALL2(pv_lock_ops.spin_lock_flags, lock, flags);
  663. }
  664. static __always_inline int __raw_spin_trylock(struct raw_spinlock *lock)
  665. {
  666. return PVOP_CALL1(int, pv_lock_ops.spin_trylock, lock);
  667. }
  668. static __always_inline void __raw_spin_unlock(struct raw_spinlock *lock)
  669. {
  670. PVOP_VCALL1(pv_lock_ops.spin_unlock, lock);
  671. }
  672. #endif
  673. #ifdef CONFIG_X86_32
  674. #define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
  675. #define PV_RESTORE_REGS "popl %edx; popl %ecx;"
  676. /* save and restore all caller-save registers, except return value */
  677. #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;"
  678. #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;"
  679. #define PV_FLAGS_ARG "0"
  680. #define PV_EXTRA_CLOBBERS
  681. #define PV_VEXTRA_CLOBBERS
  682. #else
  683. /* save and restore all caller-save registers, except return value */
  684. #define PV_SAVE_ALL_CALLER_REGS \
  685. "push %rcx;" \
  686. "push %rdx;" \
  687. "push %rsi;" \
  688. "push %rdi;" \
  689. "push %r8;" \
  690. "push %r9;" \
  691. "push %r10;" \
  692. "push %r11;"
  693. #define PV_RESTORE_ALL_CALLER_REGS \
  694. "pop %r11;" \
  695. "pop %r10;" \
  696. "pop %r9;" \
  697. "pop %r8;" \
  698. "pop %rdi;" \
  699. "pop %rsi;" \
  700. "pop %rdx;" \
  701. "pop %rcx;"
  702. /* We save some registers, but all of them, that's too much. We clobber all
  703. * caller saved registers but the argument parameter */
  704. #define PV_SAVE_REGS "pushq %%rdi;"
  705. #define PV_RESTORE_REGS "popq %%rdi;"
  706. #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
  707. #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
  708. #define PV_FLAGS_ARG "D"
  709. #endif
  710. /*
  711. * Generate a thunk around a function which saves all caller-save
  712. * registers except for the return value. This allows C functions to
  713. * be called from assembler code where fewer than normal registers are
  714. * available. It may also help code generation around calls from C
  715. * code if the common case doesn't use many registers.
  716. *
  717. * When a callee is wrapped in a thunk, the caller can assume that all
  718. * arg regs and all scratch registers are preserved across the
  719. * call. The return value in rax/eax will not be saved, even for void
  720. * functions.
  721. */
  722. #define PV_CALLEE_SAVE_REGS_THUNK(func) \
  723. extern typeof(func) __raw_callee_save_##func; \
  724. static void *__##func##__ __used = func; \
  725. \
  726. asm(".pushsection .text;" \
  727. "__raw_callee_save_" #func ": " \
  728. PV_SAVE_ALL_CALLER_REGS \
  729. "call " #func ";" \
  730. PV_RESTORE_ALL_CALLER_REGS \
  731. "ret;" \
  732. ".popsection")
  733. /* Get a reference to a callee-save function */
  734. #define PV_CALLEE_SAVE(func) \
  735. ((struct paravirt_callee_save) { __raw_callee_save_##func })
  736. /* Promise that "func" already uses the right calling convention */
  737. #define __PV_IS_CALLEE_SAVE(func) \
  738. ((struct paravirt_callee_save) { func })
  739. static inline unsigned long __raw_local_save_flags(void)
  740. {
  741. unsigned long f;
  742. asm volatile(paravirt_alt(PARAVIRT_CALL)
  743. : "=a"(f)
  744. : paravirt_type(pv_irq_ops.save_fl),
  745. paravirt_clobber(CLBR_EAX)
  746. : "memory", "cc");
  747. return f;
  748. }
  749. static inline void raw_local_irq_restore(unsigned long f)
  750. {
  751. asm volatile(paravirt_alt(PARAVIRT_CALL)
  752. : "=a"(f)
  753. : PV_FLAGS_ARG(f),
  754. paravirt_type(pv_irq_ops.restore_fl),
  755. paravirt_clobber(CLBR_EAX)
  756. : "memory", "cc");
  757. }
  758. static inline void raw_local_irq_disable(void)
  759. {
  760. asm volatile(paravirt_alt(PARAVIRT_CALL)
  761. :
  762. : paravirt_type(pv_irq_ops.irq_disable),
  763. paravirt_clobber(CLBR_EAX)
  764. : "memory", "eax", "cc");
  765. }
  766. static inline void raw_local_irq_enable(void)
  767. {
  768. asm volatile(paravirt_alt(PARAVIRT_CALL)
  769. :
  770. : paravirt_type(pv_irq_ops.irq_enable),
  771. paravirt_clobber(CLBR_EAX)
  772. : "memory", "eax", "cc");
  773. }
  774. static inline unsigned long __raw_local_irq_save(void)
  775. {
  776. unsigned long f;
  777. f = __raw_local_save_flags();
  778. raw_local_irq_disable();
  779. return f;
  780. }
  781. /* Make sure as little as possible of this mess escapes. */
  782. #undef PARAVIRT_CALL
  783. #undef __PVOP_CALL
  784. #undef __PVOP_VCALL
  785. #undef PVOP_VCALL0
  786. #undef PVOP_CALL0
  787. #undef PVOP_VCALL1
  788. #undef PVOP_CALL1
  789. #undef PVOP_VCALL2
  790. #undef PVOP_CALL2
  791. #undef PVOP_VCALL3
  792. #undef PVOP_CALL3
  793. #undef PVOP_VCALL4
  794. #undef PVOP_CALL4
  795. #else /* __ASSEMBLY__ */
  796. #define _PVSITE(ptype, clobbers, ops, word, algn) \
  797. 771:; \
  798. ops; \
  799. 772:; \
  800. .pushsection .parainstructions,"a"; \
  801. .align algn; \
  802. word 771b; \
  803. .byte ptype; \
  804. .byte 772b-771b; \
  805. .short clobbers; \
  806. .popsection
  807. #define COND_PUSH(set, mask, reg) \
  808. .if ((~(set)) & mask); push %reg; .endif
  809. #define COND_POP(set, mask, reg) \
  810. .if ((~(set)) & mask); pop %reg; .endif
  811. #ifdef CONFIG_X86_64
  812. #define PV_SAVE_REGS(set) \
  813. COND_PUSH(set, CLBR_RAX, rax); \
  814. COND_PUSH(set, CLBR_RCX, rcx); \
  815. COND_PUSH(set, CLBR_RDX, rdx); \
  816. COND_PUSH(set, CLBR_RSI, rsi); \
  817. COND_PUSH(set, CLBR_RDI, rdi); \
  818. COND_PUSH(set, CLBR_R8, r8); \
  819. COND_PUSH(set, CLBR_R9, r9); \
  820. COND_PUSH(set, CLBR_R10, r10); \
  821. COND_PUSH(set, CLBR_R11, r11)
  822. #define PV_RESTORE_REGS(set) \
  823. COND_POP(set, CLBR_R11, r11); \
  824. COND_POP(set, CLBR_R10, r10); \
  825. COND_POP(set, CLBR_R9, r9); \
  826. COND_POP(set, CLBR_R8, r8); \
  827. COND_POP(set, CLBR_RDI, rdi); \
  828. COND_POP(set, CLBR_RSI, rsi); \
  829. COND_POP(set, CLBR_RDX, rdx); \
  830. COND_POP(set, CLBR_RCX, rcx); \
  831. COND_POP(set, CLBR_RAX, rax)
  832. #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8)
  833. #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
  834. #define PARA_INDIRECT(addr) *addr(%rip)
  835. #else
  836. #define PV_SAVE_REGS(set) \
  837. COND_PUSH(set, CLBR_EAX, eax); \
  838. COND_PUSH(set, CLBR_EDI, edi); \
  839. COND_PUSH(set, CLBR_ECX, ecx); \
  840. COND_PUSH(set, CLBR_EDX, edx)
  841. #define PV_RESTORE_REGS(set) \
  842. COND_POP(set, CLBR_EDX, edx); \
  843. COND_POP(set, CLBR_ECX, ecx); \
  844. COND_POP(set, CLBR_EDI, edi); \
  845. COND_POP(set, CLBR_EAX, eax)
  846. #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
  847. #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
  848. #define PARA_INDIRECT(addr) *%cs:addr
  849. #endif
  850. #define INTERRUPT_RETURN \
  851. PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \
  852. jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
  853. #define DISABLE_INTERRUPTS(clobbers) \
  854. PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
  855. PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
  856. call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \
  857. PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
  858. #define ENABLE_INTERRUPTS(clobbers) \
  859. PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
  860. PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
  861. call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \
  862. PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
  863. #define USERGS_SYSRET32 \
  864. PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32), \
  865. CLBR_NONE, \
  866. jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32))
  867. #ifdef CONFIG_X86_32
  868. #define GET_CR0_INTO_EAX \
  869. push %ecx; push %edx; \
  870. call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
  871. pop %edx; pop %ecx
  872. #define ENABLE_INTERRUPTS_SYSEXIT \
  873. PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \
  874. CLBR_NONE, \
  875. jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
  876. #else /* !CONFIG_X86_32 */
  877. /*
  878. * If swapgs is used while the userspace stack is still current,
  879. * there's no way to call a pvop. The PV replacement *must* be
  880. * inlined, or the swapgs instruction must be trapped and emulated.
  881. */
  882. #define SWAPGS_UNSAFE_STACK \
  883. PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
  884. swapgs)
  885. /*
  886. * Note: swapgs is very special, and in practise is either going to be
  887. * implemented with a single "swapgs" instruction or something very
  888. * special. Either way, we don't need to save any registers for
  889. * it.
  890. */
  891. #define SWAPGS \
  892. PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
  893. call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \
  894. )
  895. #define GET_CR2_INTO_RCX \
  896. call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); \
  897. movq %rax, %rcx; \
  898. xorq %rax, %rax;
  899. #define PARAVIRT_ADJUST_EXCEPTION_FRAME \
  900. PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
  901. CLBR_NONE, \
  902. call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
  903. #define USERGS_SYSRET64 \
  904. PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64), \
  905. CLBR_NONE, \
  906. jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
  907. #define ENABLE_INTERRUPTS_SYSEXIT32 \
  908. PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \
  909. CLBR_NONE, \
  910. jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
  911. #endif /* CONFIG_X86_32 */
  912. #endif /* __ASSEMBLY__ */
  913. #endif /* CONFIG_PARAVIRT */
  914. #endif /* _ASM_X86_PARAVIRT_H */