svm.c 47 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * AMD SVM support
  5. *
  6. * Copyright (C) 2006 Qumranet, Inc.
  7. *
  8. * Authors:
  9. * Yaniv Kamay <yaniv@qumranet.com>
  10. * Avi Kivity <avi@qumranet.com>
  11. *
  12. * This work is licensed under the terms of the GNU GPL, version 2. See
  13. * the COPYING file in the top-level directory.
  14. *
  15. */
  16. #include <linux/kvm_host.h>
  17. #include "kvm_svm.h"
  18. #include "irq.h"
  19. #include "mmu.h"
  20. #include <linux/module.h>
  21. #include <linux/kernel.h>
  22. #include <linux/vmalloc.h>
  23. #include <linux/highmem.h>
  24. #include <linux/sched.h>
  25. #include <asm/desc.h>
  26. MODULE_AUTHOR("Qumranet");
  27. MODULE_LICENSE("GPL");
  28. #define IOPM_ALLOC_ORDER 2
  29. #define MSRPM_ALLOC_ORDER 1
  30. #define DB_VECTOR 1
  31. #define UD_VECTOR 6
  32. #define GP_VECTOR 13
  33. #define DR7_GD_MASK (1 << 13)
  34. #define DR6_BD_MASK (1 << 13)
  35. #define SEG_TYPE_LDT 2
  36. #define SEG_TYPE_BUSY_TSS16 3
  37. #define SVM_FEATURE_NPT (1 << 0)
  38. #define SVM_FEATURE_LBRV (1 << 1)
  39. #define SVM_DEATURE_SVML (1 << 2)
  40. #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  41. /* enable NPT for AMD64 and X86 with PAE */
  42. #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
  43. static bool npt_enabled = true;
  44. #else
  45. static bool npt_enabled = false;
  46. #endif
  47. static int npt = 1;
  48. module_param(npt, int, S_IRUGO);
  49. static void kvm_reput_irq(struct vcpu_svm *svm);
  50. static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
  51. {
  52. return container_of(vcpu, struct vcpu_svm, vcpu);
  53. }
  54. static unsigned long iopm_base;
  55. struct kvm_ldttss_desc {
  56. u16 limit0;
  57. u16 base0;
  58. unsigned base1 : 8, type : 5, dpl : 2, p : 1;
  59. unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8;
  60. u32 base3;
  61. u32 zero1;
  62. } __attribute__((packed));
  63. struct svm_cpu_data {
  64. int cpu;
  65. u64 asid_generation;
  66. u32 max_asid;
  67. u32 next_asid;
  68. struct kvm_ldttss_desc *tss_desc;
  69. struct page *save_area;
  70. };
  71. static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
  72. static uint32_t svm_features;
  73. struct svm_init_data {
  74. int cpu;
  75. int r;
  76. };
  77. static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
  78. #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
  79. #define MSRS_RANGE_SIZE 2048
  80. #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
  81. #define MAX_INST_SIZE 15
  82. static inline u32 svm_has(u32 feat)
  83. {
  84. return svm_features & feat;
  85. }
  86. static inline u8 pop_irq(struct kvm_vcpu *vcpu)
  87. {
  88. int word_index = __ffs(vcpu->arch.irq_summary);
  89. int bit_index = __ffs(vcpu->arch.irq_pending[word_index]);
  90. int irq = word_index * BITS_PER_LONG + bit_index;
  91. clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]);
  92. if (!vcpu->arch.irq_pending[word_index])
  93. clear_bit(word_index, &vcpu->arch.irq_summary);
  94. return irq;
  95. }
  96. static inline void push_irq(struct kvm_vcpu *vcpu, u8 irq)
  97. {
  98. set_bit(irq, vcpu->arch.irq_pending);
  99. set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary);
  100. }
  101. static inline void clgi(void)
  102. {
  103. asm volatile (SVM_CLGI);
  104. }
  105. static inline void stgi(void)
  106. {
  107. asm volatile (SVM_STGI);
  108. }
  109. static inline void invlpga(unsigned long addr, u32 asid)
  110. {
  111. asm volatile (SVM_INVLPGA :: "a"(addr), "c"(asid));
  112. }
  113. static inline unsigned long kvm_read_cr2(void)
  114. {
  115. unsigned long cr2;
  116. asm volatile ("mov %%cr2, %0" : "=r" (cr2));
  117. return cr2;
  118. }
  119. static inline void kvm_write_cr2(unsigned long val)
  120. {
  121. asm volatile ("mov %0, %%cr2" :: "r" (val));
  122. }
  123. static inline unsigned long read_dr6(void)
  124. {
  125. unsigned long dr6;
  126. asm volatile ("mov %%dr6, %0" : "=r" (dr6));
  127. return dr6;
  128. }
  129. static inline void write_dr6(unsigned long val)
  130. {
  131. asm volatile ("mov %0, %%dr6" :: "r" (val));
  132. }
  133. static inline unsigned long read_dr7(void)
  134. {
  135. unsigned long dr7;
  136. asm volatile ("mov %%dr7, %0" : "=r" (dr7));
  137. return dr7;
  138. }
  139. static inline void write_dr7(unsigned long val)
  140. {
  141. asm volatile ("mov %0, %%dr7" :: "r" (val));
  142. }
  143. static inline void force_new_asid(struct kvm_vcpu *vcpu)
  144. {
  145. to_svm(vcpu)->asid_generation--;
  146. }
  147. static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
  148. {
  149. force_new_asid(vcpu);
  150. }
  151. static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
  152. {
  153. if (!npt_enabled && !(efer & EFER_LMA))
  154. efer &= ~EFER_LME;
  155. to_svm(vcpu)->vmcb->save.efer = efer | MSR_EFER_SVME_MASK;
  156. vcpu->arch.shadow_efer = efer;
  157. }
  158. static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
  159. bool has_error_code, u32 error_code)
  160. {
  161. struct vcpu_svm *svm = to_svm(vcpu);
  162. svm->vmcb->control.event_inj = nr
  163. | SVM_EVTINJ_VALID
  164. | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
  165. | SVM_EVTINJ_TYPE_EXEPT;
  166. svm->vmcb->control.event_inj_err = error_code;
  167. }
  168. static bool svm_exception_injected(struct kvm_vcpu *vcpu)
  169. {
  170. struct vcpu_svm *svm = to_svm(vcpu);
  171. return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID);
  172. }
  173. static int is_external_interrupt(u32 info)
  174. {
  175. info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
  176. return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
  177. }
  178. static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
  179. {
  180. struct vcpu_svm *svm = to_svm(vcpu);
  181. if (!svm->next_rip) {
  182. printk(KERN_DEBUG "%s: NOP\n", __func__);
  183. return;
  184. }
  185. if (svm->next_rip - svm->vmcb->save.rip > MAX_INST_SIZE)
  186. printk(KERN_ERR "%s: ip 0x%llx next 0x%llx\n",
  187. __func__,
  188. svm->vmcb->save.rip,
  189. svm->next_rip);
  190. vcpu->arch.rip = svm->vmcb->save.rip = svm->next_rip;
  191. svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
  192. vcpu->arch.interrupt_window_open = 1;
  193. }
  194. static int has_svm(void)
  195. {
  196. uint32_t eax, ebx, ecx, edx;
  197. if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
  198. printk(KERN_INFO "has_svm: not amd\n");
  199. return 0;
  200. }
  201. cpuid(0x80000000, &eax, &ebx, &ecx, &edx);
  202. if (eax < SVM_CPUID_FUNC) {
  203. printk(KERN_INFO "has_svm: can't execute cpuid_8000000a\n");
  204. return 0;
  205. }
  206. cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
  207. if (!(ecx & (1 << SVM_CPUID_FEATURE_SHIFT))) {
  208. printk(KERN_DEBUG "has_svm: svm not available\n");
  209. return 0;
  210. }
  211. return 1;
  212. }
  213. static void svm_hardware_disable(void *garbage)
  214. {
  215. struct svm_cpu_data *svm_data
  216. = per_cpu(svm_data, raw_smp_processor_id());
  217. if (svm_data) {
  218. uint64_t efer;
  219. wrmsrl(MSR_VM_HSAVE_PA, 0);
  220. rdmsrl(MSR_EFER, efer);
  221. wrmsrl(MSR_EFER, efer & ~MSR_EFER_SVME_MASK);
  222. per_cpu(svm_data, raw_smp_processor_id()) = NULL;
  223. __free_page(svm_data->save_area);
  224. kfree(svm_data);
  225. }
  226. }
  227. static void svm_hardware_enable(void *garbage)
  228. {
  229. struct svm_cpu_data *svm_data;
  230. uint64_t efer;
  231. struct desc_ptr gdt_descr;
  232. struct desc_struct *gdt;
  233. int me = raw_smp_processor_id();
  234. if (!has_svm()) {
  235. printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me);
  236. return;
  237. }
  238. svm_data = per_cpu(svm_data, me);
  239. if (!svm_data) {
  240. printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n",
  241. me);
  242. return;
  243. }
  244. svm_data->asid_generation = 1;
  245. svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
  246. svm_data->next_asid = svm_data->max_asid + 1;
  247. asm volatile ("sgdt %0" : "=m"(gdt_descr));
  248. gdt = (struct desc_struct *)gdt_descr.address;
  249. svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
  250. rdmsrl(MSR_EFER, efer);
  251. wrmsrl(MSR_EFER, efer | MSR_EFER_SVME_MASK);
  252. wrmsrl(MSR_VM_HSAVE_PA,
  253. page_to_pfn(svm_data->save_area) << PAGE_SHIFT);
  254. }
  255. static int svm_cpu_init(int cpu)
  256. {
  257. struct svm_cpu_data *svm_data;
  258. int r;
  259. svm_data = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
  260. if (!svm_data)
  261. return -ENOMEM;
  262. svm_data->cpu = cpu;
  263. svm_data->save_area = alloc_page(GFP_KERNEL);
  264. r = -ENOMEM;
  265. if (!svm_data->save_area)
  266. goto err_1;
  267. per_cpu(svm_data, cpu) = svm_data;
  268. return 0;
  269. err_1:
  270. kfree(svm_data);
  271. return r;
  272. }
  273. static void set_msr_interception(u32 *msrpm, unsigned msr,
  274. int read, int write)
  275. {
  276. int i;
  277. for (i = 0; i < NUM_MSR_MAPS; i++) {
  278. if (msr >= msrpm_ranges[i] &&
  279. msr < msrpm_ranges[i] + MSRS_IN_RANGE) {
  280. u32 msr_offset = (i * MSRS_IN_RANGE + msr -
  281. msrpm_ranges[i]) * 2;
  282. u32 *base = msrpm + (msr_offset / 32);
  283. u32 msr_shift = msr_offset % 32;
  284. u32 mask = ((write) ? 0 : 2) | ((read) ? 0 : 1);
  285. *base = (*base & ~(0x3 << msr_shift)) |
  286. (mask << msr_shift);
  287. return;
  288. }
  289. }
  290. BUG();
  291. }
  292. static void svm_vcpu_init_msrpm(u32 *msrpm)
  293. {
  294. memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
  295. #ifdef CONFIG_X86_64
  296. set_msr_interception(msrpm, MSR_GS_BASE, 1, 1);
  297. set_msr_interception(msrpm, MSR_FS_BASE, 1, 1);
  298. set_msr_interception(msrpm, MSR_KERNEL_GS_BASE, 1, 1);
  299. set_msr_interception(msrpm, MSR_LSTAR, 1, 1);
  300. set_msr_interception(msrpm, MSR_CSTAR, 1, 1);
  301. set_msr_interception(msrpm, MSR_SYSCALL_MASK, 1, 1);
  302. #endif
  303. set_msr_interception(msrpm, MSR_K6_STAR, 1, 1);
  304. set_msr_interception(msrpm, MSR_IA32_SYSENTER_CS, 1, 1);
  305. set_msr_interception(msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
  306. set_msr_interception(msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
  307. }
  308. static void svm_enable_lbrv(struct vcpu_svm *svm)
  309. {
  310. u32 *msrpm = svm->msrpm;
  311. svm->vmcb->control.lbr_ctl = 1;
  312. set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
  313. set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
  314. set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
  315. set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
  316. }
  317. static void svm_disable_lbrv(struct vcpu_svm *svm)
  318. {
  319. u32 *msrpm = svm->msrpm;
  320. svm->vmcb->control.lbr_ctl = 0;
  321. set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
  322. set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
  323. set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
  324. set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
  325. }
  326. static __init int svm_hardware_setup(void)
  327. {
  328. int cpu;
  329. struct page *iopm_pages;
  330. void *iopm_va;
  331. int r;
  332. iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
  333. if (!iopm_pages)
  334. return -ENOMEM;
  335. iopm_va = page_address(iopm_pages);
  336. memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
  337. clear_bit(0x80, iopm_va); /* allow direct access to PC debug port */
  338. iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
  339. if (boot_cpu_has(X86_FEATURE_NX))
  340. kvm_enable_efer_bits(EFER_NX);
  341. for_each_online_cpu(cpu) {
  342. r = svm_cpu_init(cpu);
  343. if (r)
  344. goto err;
  345. }
  346. svm_features = cpuid_edx(SVM_CPUID_FUNC);
  347. if (!svm_has(SVM_FEATURE_NPT))
  348. npt_enabled = false;
  349. if (npt_enabled && !npt) {
  350. printk(KERN_INFO "kvm: Nested Paging disabled\n");
  351. npt_enabled = false;
  352. }
  353. if (npt_enabled) {
  354. printk(KERN_INFO "kvm: Nested Paging enabled\n");
  355. kvm_enable_tdp();
  356. }
  357. return 0;
  358. err:
  359. __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
  360. iopm_base = 0;
  361. return r;
  362. }
  363. static __exit void svm_hardware_unsetup(void)
  364. {
  365. __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
  366. iopm_base = 0;
  367. }
  368. static void init_seg(struct vmcb_seg *seg)
  369. {
  370. seg->selector = 0;
  371. seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
  372. SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
  373. seg->limit = 0xffff;
  374. seg->base = 0;
  375. }
  376. static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
  377. {
  378. seg->selector = 0;
  379. seg->attrib = SVM_SELECTOR_P_MASK | type;
  380. seg->limit = 0xffff;
  381. seg->base = 0;
  382. }
  383. static void init_vmcb(struct vcpu_svm *svm)
  384. {
  385. struct vmcb_control_area *control = &svm->vmcb->control;
  386. struct vmcb_save_area *save = &svm->vmcb->save;
  387. control->intercept_cr_read = INTERCEPT_CR0_MASK |
  388. INTERCEPT_CR3_MASK |
  389. INTERCEPT_CR4_MASK |
  390. INTERCEPT_CR8_MASK;
  391. control->intercept_cr_write = INTERCEPT_CR0_MASK |
  392. INTERCEPT_CR3_MASK |
  393. INTERCEPT_CR4_MASK |
  394. INTERCEPT_CR8_MASK;
  395. control->intercept_dr_read = INTERCEPT_DR0_MASK |
  396. INTERCEPT_DR1_MASK |
  397. INTERCEPT_DR2_MASK |
  398. INTERCEPT_DR3_MASK;
  399. control->intercept_dr_write = INTERCEPT_DR0_MASK |
  400. INTERCEPT_DR1_MASK |
  401. INTERCEPT_DR2_MASK |
  402. INTERCEPT_DR3_MASK |
  403. INTERCEPT_DR5_MASK |
  404. INTERCEPT_DR7_MASK;
  405. control->intercept_exceptions = (1 << PF_VECTOR) |
  406. (1 << UD_VECTOR) |
  407. (1 << MC_VECTOR);
  408. control->intercept = (1ULL << INTERCEPT_INTR) |
  409. (1ULL << INTERCEPT_NMI) |
  410. (1ULL << INTERCEPT_SMI) |
  411. /*
  412. * selective cr0 intercept bug?
  413. * 0: 0f 22 d8 mov %eax,%cr3
  414. * 3: 0f 20 c0 mov %cr0,%eax
  415. * 6: 0d 00 00 00 80 or $0x80000000,%eax
  416. * b: 0f 22 c0 mov %eax,%cr0
  417. * set cr3 ->interception
  418. * get cr0 ->interception
  419. * set cr0 -> no interception
  420. */
  421. /* (1ULL << INTERCEPT_SELECTIVE_CR0) | */
  422. (1ULL << INTERCEPT_CPUID) |
  423. (1ULL << INTERCEPT_INVD) |
  424. (1ULL << INTERCEPT_HLT) |
  425. (1ULL << INTERCEPT_INVLPGA) |
  426. (1ULL << INTERCEPT_IOIO_PROT) |
  427. (1ULL << INTERCEPT_MSR_PROT) |
  428. (1ULL << INTERCEPT_TASK_SWITCH) |
  429. (1ULL << INTERCEPT_SHUTDOWN) |
  430. (1ULL << INTERCEPT_VMRUN) |
  431. (1ULL << INTERCEPT_VMMCALL) |
  432. (1ULL << INTERCEPT_VMLOAD) |
  433. (1ULL << INTERCEPT_VMSAVE) |
  434. (1ULL << INTERCEPT_STGI) |
  435. (1ULL << INTERCEPT_CLGI) |
  436. (1ULL << INTERCEPT_SKINIT) |
  437. (1ULL << INTERCEPT_WBINVD) |
  438. (1ULL << INTERCEPT_MONITOR) |
  439. (1ULL << INTERCEPT_MWAIT);
  440. control->iopm_base_pa = iopm_base;
  441. control->msrpm_base_pa = __pa(svm->msrpm);
  442. control->tsc_offset = 0;
  443. control->int_ctl = V_INTR_MASKING_MASK;
  444. init_seg(&save->es);
  445. init_seg(&save->ss);
  446. init_seg(&save->ds);
  447. init_seg(&save->fs);
  448. init_seg(&save->gs);
  449. save->cs.selector = 0xf000;
  450. /* Executable/Readable Code Segment */
  451. save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
  452. SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
  453. save->cs.limit = 0xffff;
  454. /*
  455. * cs.base should really be 0xffff0000, but vmx can't handle that, so
  456. * be consistent with it.
  457. *
  458. * Replace when we have real mode working for vmx.
  459. */
  460. save->cs.base = 0xf0000;
  461. save->gdtr.limit = 0xffff;
  462. save->idtr.limit = 0xffff;
  463. init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
  464. init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
  465. save->efer = MSR_EFER_SVME_MASK;
  466. save->dr6 = 0xffff0ff0;
  467. save->dr7 = 0x400;
  468. save->rflags = 2;
  469. save->rip = 0x0000fff0;
  470. /*
  471. * cr0 val on cpu init should be 0x60000010, we enable cpu
  472. * cache by default. the orderly way is to enable cache in bios.
  473. */
  474. save->cr0 = 0x00000010 | X86_CR0_PG | X86_CR0_WP;
  475. save->cr4 = X86_CR4_PAE;
  476. /* rdx = ?? */
  477. if (npt_enabled) {
  478. /* Setup VMCB for Nested Paging */
  479. control->nested_ctl = 1;
  480. control->intercept_exceptions &= ~(1 << PF_VECTOR);
  481. control->intercept_cr_read &= ~(INTERCEPT_CR0_MASK|
  482. INTERCEPT_CR3_MASK);
  483. control->intercept_cr_write &= ~(INTERCEPT_CR0_MASK|
  484. INTERCEPT_CR3_MASK);
  485. save->g_pat = 0x0007040600070406ULL;
  486. /* enable caching because the QEMU Bios doesn't enable it */
  487. save->cr0 = X86_CR0_ET;
  488. save->cr3 = 0;
  489. save->cr4 = 0;
  490. }
  491. }
  492. static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
  493. {
  494. struct vcpu_svm *svm = to_svm(vcpu);
  495. init_vmcb(svm);
  496. if (vcpu->vcpu_id != 0) {
  497. svm->vmcb->save.rip = 0;
  498. svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;
  499. svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;
  500. }
  501. return 0;
  502. }
  503. static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
  504. {
  505. struct vcpu_svm *svm;
  506. struct page *page;
  507. struct page *msrpm_pages;
  508. int err;
  509. svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
  510. if (!svm) {
  511. err = -ENOMEM;
  512. goto out;
  513. }
  514. err = kvm_vcpu_init(&svm->vcpu, kvm, id);
  515. if (err)
  516. goto free_svm;
  517. page = alloc_page(GFP_KERNEL);
  518. if (!page) {
  519. err = -ENOMEM;
  520. goto uninit;
  521. }
  522. err = -ENOMEM;
  523. msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
  524. if (!msrpm_pages)
  525. goto uninit;
  526. svm->msrpm = page_address(msrpm_pages);
  527. svm_vcpu_init_msrpm(svm->msrpm);
  528. svm->vmcb = page_address(page);
  529. clear_page(svm->vmcb);
  530. svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
  531. svm->asid_generation = 0;
  532. memset(svm->db_regs, 0, sizeof(svm->db_regs));
  533. init_vmcb(svm);
  534. fx_init(&svm->vcpu);
  535. svm->vcpu.fpu_active = 1;
  536. svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
  537. if (svm->vcpu.vcpu_id == 0)
  538. svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
  539. return &svm->vcpu;
  540. uninit:
  541. kvm_vcpu_uninit(&svm->vcpu);
  542. free_svm:
  543. kmem_cache_free(kvm_vcpu_cache, svm);
  544. out:
  545. return ERR_PTR(err);
  546. }
  547. static void svm_free_vcpu(struct kvm_vcpu *vcpu)
  548. {
  549. struct vcpu_svm *svm = to_svm(vcpu);
  550. __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
  551. __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
  552. kvm_vcpu_uninit(vcpu);
  553. kmem_cache_free(kvm_vcpu_cache, svm);
  554. }
  555. static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  556. {
  557. struct vcpu_svm *svm = to_svm(vcpu);
  558. int i;
  559. if (unlikely(cpu != vcpu->cpu)) {
  560. u64 tsc_this, delta;
  561. /*
  562. * Make sure that the guest sees a monotonically
  563. * increasing TSC.
  564. */
  565. rdtscll(tsc_this);
  566. delta = vcpu->arch.host_tsc - tsc_this;
  567. svm->vmcb->control.tsc_offset += delta;
  568. vcpu->cpu = cpu;
  569. kvm_migrate_apic_timer(vcpu);
  570. }
  571. for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
  572. rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
  573. }
  574. static void svm_vcpu_put(struct kvm_vcpu *vcpu)
  575. {
  576. struct vcpu_svm *svm = to_svm(vcpu);
  577. int i;
  578. ++vcpu->stat.host_state_reload;
  579. for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
  580. wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
  581. rdtscll(vcpu->arch.host_tsc);
  582. }
  583. static void svm_vcpu_decache(struct kvm_vcpu *vcpu)
  584. {
  585. }
  586. static void svm_cache_regs(struct kvm_vcpu *vcpu)
  587. {
  588. struct vcpu_svm *svm = to_svm(vcpu);
  589. vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
  590. vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
  591. vcpu->arch.rip = svm->vmcb->save.rip;
  592. }
  593. static void svm_decache_regs(struct kvm_vcpu *vcpu)
  594. {
  595. struct vcpu_svm *svm = to_svm(vcpu);
  596. svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
  597. svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
  598. svm->vmcb->save.rip = vcpu->arch.rip;
  599. }
  600. static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
  601. {
  602. return to_svm(vcpu)->vmcb->save.rflags;
  603. }
  604. static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  605. {
  606. to_svm(vcpu)->vmcb->save.rflags = rflags;
  607. }
  608. static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
  609. {
  610. struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
  611. switch (seg) {
  612. case VCPU_SREG_CS: return &save->cs;
  613. case VCPU_SREG_DS: return &save->ds;
  614. case VCPU_SREG_ES: return &save->es;
  615. case VCPU_SREG_FS: return &save->fs;
  616. case VCPU_SREG_GS: return &save->gs;
  617. case VCPU_SREG_SS: return &save->ss;
  618. case VCPU_SREG_TR: return &save->tr;
  619. case VCPU_SREG_LDTR: return &save->ldtr;
  620. }
  621. BUG();
  622. return NULL;
  623. }
  624. static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
  625. {
  626. struct vmcb_seg *s = svm_seg(vcpu, seg);
  627. return s->base;
  628. }
  629. static void svm_get_segment(struct kvm_vcpu *vcpu,
  630. struct kvm_segment *var, int seg)
  631. {
  632. struct vmcb_seg *s = svm_seg(vcpu, seg);
  633. var->base = s->base;
  634. var->limit = s->limit;
  635. var->selector = s->selector;
  636. var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
  637. var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
  638. var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
  639. var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
  640. var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
  641. var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
  642. var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
  643. var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1;
  644. var->unusable = !var->present;
  645. }
  646. static int svm_get_cpl(struct kvm_vcpu *vcpu)
  647. {
  648. struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
  649. return save->cpl;
  650. }
  651. static void svm_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  652. {
  653. struct vcpu_svm *svm = to_svm(vcpu);
  654. dt->limit = svm->vmcb->save.idtr.limit;
  655. dt->base = svm->vmcb->save.idtr.base;
  656. }
  657. static void svm_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  658. {
  659. struct vcpu_svm *svm = to_svm(vcpu);
  660. svm->vmcb->save.idtr.limit = dt->limit;
  661. svm->vmcb->save.idtr.base = dt->base ;
  662. }
  663. static void svm_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  664. {
  665. struct vcpu_svm *svm = to_svm(vcpu);
  666. dt->limit = svm->vmcb->save.gdtr.limit;
  667. dt->base = svm->vmcb->save.gdtr.base;
  668. }
  669. static void svm_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  670. {
  671. struct vcpu_svm *svm = to_svm(vcpu);
  672. svm->vmcb->save.gdtr.limit = dt->limit;
  673. svm->vmcb->save.gdtr.base = dt->base ;
  674. }
  675. static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
  676. {
  677. }
  678. static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  679. {
  680. struct vcpu_svm *svm = to_svm(vcpu);
  681. #ifdef CONFIG_X86_64
  682. if (vcpu->arch.shadow_efer & EFER_LME) {
  683. if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
  684. vcpu->arch.shadow_efer |= EFER_LMA;
  685. svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
  686. }
  687. if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
  688. vcpu->arch.shadow_efer &= ~EFER_LMA;
  689. svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
  690. }
  691. }
  692. #endif
  693. if (npt_enabled)
  694. goto set;
  695. if ((vcpu->arch.cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) {
  696. svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR);
  697. vcpu->fpu_active = 1;
  698. }
  699. vcpu->arch.cr0 = cr0;
  700. cr0 |= X86_CR0_PG | X86_CR0_WP;
  701. if (!vcpu->fpu_active) {
  702. svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR);
  703. cr0 |= X86_CR0_TS;
  704. }
  705. set:
  706. /*
  707. * re-enable caching here because the QEMU bios
  708. * does not do it - this results in some delay at
  709. * reboot
  710. */
  711. cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
  712. svm->vmcb->save.cr0 = cr0;
  713. }
  714. static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  715. {
  716. unsigned long host_cr4_mce = read_cr4() & X86_CR4_MCE;
  717. vcpu->arch.cr4 = cr4;
  718. if (!npt_enabled)
  719. cr4 |= X86_CR4_PAE;
  720. cr4 |= host_cr4_mce;
  721. to_svm(vcpu)->vmcb->save.cr4 = cr4;
  722. }
  723. static void svm_set_segment(struct kvm_vcpu *vcpu,
  724. struct kvm_segment *var, int seg)
  725. {
  726. struct vcpu_svm *svm = to_svm(vcpu);
  727. struct vmcb_seg *s = svm_seg(vcpu, seg);
  728. s->base = var->base;
  729. s->limit = var->limit;
  730. s->selector = var->selector;
  731. if (var->unusable)
  732. s->attrib = 0;
  733. else {
  734. s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
  735. s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
  736. s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
  737. s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
  738. s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
  739. s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
  740. s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
  741. s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
  742. }
  743. if (seg == VCPU_SREG_CS)
  744. svm->vmcb->save.cpl
  745. = (svm->vmcb->save.cs.attrib
  746. >> SVM_SELECTOR_DPL_SHIFT) & 3;
  747. }
  748. /* FIXME:
  749. svm(vcpu)->vmcb->control.int_ctl &= ~V_TPR_MASK;
  750. svm(vcpu)->vmcb->control.int_ctl |= (sregs->cr8 & V_TPR_MASK);
  751. */
  752. static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg)
  753. {
  754. return -EOPNOTSUPP;
  755. }
  756. static int svm_get_irq(struct kvm_vcpu *vcpu)
  757. {
  758. struct vcpu_svm *svm = to_svm(vcpu);
  759. u32 exit_int_info = svm->vmcb->control.exit_int_info;
  760. if (is_external_interrupt(exit_int_info))
  761. return exit_int_info & SVM_EVTINJ_VEC_MASK;
  762. return -1;
  763. }
  764. static void load_host_msrs(struct kvm_vcpu *vcpu)
  765. {
  766. #ifdef CONFIG_X86_64
  767. wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
  768. #endif
  769. }
  770. static void save_host_msrs(struct kvm_vcpu *vcpu)
  771. {
  772. #ifdef CONFIG_X86_64
  773. rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
  774. #endif
  775. }
  776. static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *svm_data)
  777. {
  778. if (svm_data->next_asid > svm_data->max_asid) {
  779. ++svm_data->asid_generation;
  780. svm_data->next_asid = 1;
  781. svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
  782. }
  783. svm->vcpu.cpu = svm_data->cpu;
  784. svm->asid_generation = svm_data->asid_generation;
  785. svm->vmcb->control.asid = svm_data->next_asid++;
  786. }
  787. static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr)
  788. {
  789. return to_svm(vcpu)->db_regs[dr];
  790. }
  791. static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value,
  792. int *exception)
  793. {
  794. struct vcpu_svm *svm = to_svm(vcpu);
  795. *exception = 0;
  796. if (svm->vmcb->save.dr7 & DR7_GD_MASK) {
  797. svm->vmcb->save.dr7 &= ~DR7_GD_MASK;
  798. svm->vmcb->save.dr6 |= DR6_BD_MASK;
  799. *exception = DB_VECTOR;
  800. return;
  801. }
  802. switch (dr) {
  803. case 0 ... 3:
  804. svm->db_regs[dr] = value;
  805. return;
  806. case 4 ... 5:
  807. if (vcpu->arch.cr4 & X86_CR4_DE) {
  808. *exception = UD_VECTOR;
  809. return;
  810. }
  811. case 7: {
  812. if (value & ~((1ULL << 32) - 1)) {
  813. *exception = GP_VECTOR;
  814. return;
  815. }
  816. svm->vmcb->save.dr7 = value;
  817. return;
  818. }
  819. default:
  820. printk(KERN_DEBUG "%s: unexpected dr %u\n",
  821. __func__, dr);
  822. *exception = UD_VECTOR;
  823. return;
  824. }
  825. }
  826. static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  827. {
  828. u32 exit_int_info = svm->vmcb->control.exit_int_info;
  829. struct kvm *kvm = svm->vcpu.kvm;
  830. u64 fault_address;
  831. u32 error_code;
  832. if (!irqchip_in_kernel(kvm) &&
  833. is_external_interrupt(exit_int_info))
  834. push_irq(&svm->vcpu, exit_int_info & SVM_EVTINJ_VEC_MASK);
  835. fault_address = svm->vmcb->control.exit_info_2;
  836. error_code = svm->vmcb->control.exit_info_1;
  837. return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
  838. }
  839. static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  840. {
  841. int er;
  842. er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, EMULTYPE_TRAP_UD);
  843. if (er != EMULATE_DONE)
  844. kvm_queue_exception(&svm->vcpu, UD_VECTOR);
  845. return 1;
  846. }
  847. static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  848. {
  849. svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR);
  850. if (!(svm->vcpu.arch.cr0 & X86_CR0_TS))
  851. svm->vmcb->save.cr0 &= ~X86_CR0_TS;
  852. svm->vcpu.fpu_active = 1;
  853. return 1;
  854. }
  855. static int mc_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  856. {
  857. /*
  858. * On an #MC intercept the MCE handler is not called automatically in
  859. * the host. So do it by hand here.
  860. */
  861. asm volatile (
  862. "int $0x12\n");
  863. /* not sure if we ever come back to this point */
  864. return 1;
  865. }
  866. static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  867. {
  868. /*
  869. * VMCB is undefined after a SHUTDOWN intercept
  870. * so reinitialize it.
  871. */
  872. clear_page(svm->vmcb);
  873. init_vmcb(svm);
  874. kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
  875. return 0;
  876. }
  877. static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  878. {
  879. u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
  880. int size, down, in, string, rep;
  881. unsigned port;
  882. ++svm->vcpu.stat.io_exits;
  883. svm->next_rip = svm->vmcb->control.exit_info_2;
  884. string = (io_info & SVM_IOIO_STR_MASK) != 0;
  885. if (string) {
  886. if (emulate_instruction(&svm->vcpu,
  887. kvm_run, 0, 0, 0) == EMULATE_DO_MMIO)
  888. return 0;
  889. return 1;
  890. }
  891. in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
  892. port = io_info >> 16;
  893. size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
  894. rep = (io_info & SVM_IOIO_REP_MASK) != 0;
  895. down = (svm->vmcb->save.rflags & X86_EFLAGS_DF) != 0;
  896. return kvm_emulate_pio(&svm->vcpu, kvm_run, in, size, port);
  897. }
  898. static int nop_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  899. {
  900. return 1;
  901. }
  902. static int halt_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  903. {
  904. svm->next_rip = svm->vmcb->save.rip + 1;
  905. skip_emulated_instruction(&svm->vcpu);
  906. return kvm_emulate_halt(&svm->vcpu);
  907. }
  908. static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  909. {
  910. svm->next_rip = svm->vmcb->save.rip + 3;
  911. skip_emulated_instruction(&svm->vcpu);
  912. kvm_emulate_hypercall(&svm->vcpu);
  913. return 1;
  914. }
  915. static int invalid_op_interception(struct vcpu_svm *svm,
  916. struct kvm_run *kvm_run)
  917. {
  918. kvm_queue_exception(&svm->vcpu, UD_VECTOR);
  919. return 1;
  920. }
  921. static int task_switch_interception(struct vcpu_svm *svm,
  922. struct kvm_run *kvm_run)
  923. {
  924. u16 tss_selector;
  925. tss_selector = (u16)svm->vmcb->control.exit_info_1;
  926. if (svm->vmcb->control.exit_info_2 &
  927. (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
  928. return kvm_task_switch(&svm->vcpu, tss_selector,
  929. TASK_SWITCH_IRET);
  930. if (svm->vmcb->control.exit_info_2 &
  931. (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
  932. return kvm_task_switch(&svm->vcpu, tss_selector,
  933. TASK_SWITCH_JMP);
  934. return kvm_task_switch(&svm->vcpu, tss_selector, TASK_SWITCH_CALL);
  935. }
  936. static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  937. {
  938. svm->next_rip = svm->vmcb->save.rip + 2;
  939. kvm_emulate_cpuid(&svm->vcpu);
  940. return 1;
  941. }
  942. static int emulate_on_interception(struct vcpu_svm *svm,
  943. struct kvm_run *kvm_run)
  944. {
  945. if (emulate_instruction(&svm->vcpu, NULL, 0, 0, 0) != EMULATE_DONE)
  946. pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
  947. return 1;
  948. }
  949. static int cr8_write_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  950. {
  951. emulate_instruction(&svm->vcpu, NULL, 0, 0, 0);
  952. if (irqchip_in_kernel(svm->vcpu.kvm))
  953. return 1;
  954. kvm_run->exit_reason = KVM_EXIT_SET_TPR;
  955. return 0;
  956. }
  957. static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
  958. {
  959. struct vcpu_svm *svm = to_svm(vcpu);
  960. switch (ecx) {
  961. case MSR_IA32_TIME_STAMP_COUNTER: {
  962. u64 tsc;
  963. rdtscll(tsc);
  964. *data = svm->vmcb->control.tsc_offset + tsc;
  965. break;
  966. }
  967. case MSR_K6_STAR:
  968. *data = svm->vmcb->save.star;
  969. break;
  970. #ifdef CONFIG_X86_64
  971. case MSR_LSTAR:
  972. *data = svm->vmcb->save.lstar;
  973. break;
  974. case MSR_CSTAR:
  975. *data = svm->vmcb->save.cstar;
  976. break;
  977. case MSR_KERNEL_GS_BASE:
  978. *data = svm->vmcb->save.kernel_gs_base;
  979. break;
  980. case MSR_SYSCALL_MASK:
  981. *data = svm->vmcb->save.sfmask;
  982. break;
  983. #endif
  984. case MSR_IA32_SYSENTER_CS:
  985. *data = svm->vmcb->save.sysenter_cs;
  986. break;
  987. case MSR_IA32_SYSENTER_EIP:
  988. *data = svm->vmcb->save.sysenter_eip;
  989. break;
  990. case MSR_IA32_SYSENTER_ESP:
  991. *data = svm->vmcb->save.sysenter_esp;
  992. break;
  993. /* Nobody will change the following 5 values in the VMCB so
  994. we can safely return them on rdmsr. They will always be 0
  995. until LBRV is implemented. */
  996. case MSR_IA32_DEBUGCTLMSR:
  997. *data = svm->vmcb->save.dbgctl;
  998. break;
  999. case MSR_IA32_LASTBRANCHFROMIP:
  1000. *data = svm->vmcb->save.br_from;
  1001. break;
  1002. case MSR_IA32_LASTBRANCHTOIP:
  1003. *data = svm->vmcb->save.br_to;
  1004. break;
  1005. case MSR_IA32_LASTINTFROMIP:
  1006. *data = svm->vmcb->save.last_excp_from;
  1007. break;
  1008. case MSR_IA32_LASTINTTOIP:
  1009. *data = svm->vmcb->save.last_excp_to;
  1010. break;
  1011. default:
  1012. return kvm_get_msr_common(vcpu, ecx, data);
  1013. }
  1014. return 0;
  1015. }
  1016. static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  1017. {
  1018. u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
  1019. u64 data;
  1020. if (svm_get_msr(&svm->vcpu, ecx, &data))
  1021. kvm_inject_gp(&svm->vcpu, 0);
  1022. else {
  1023. svm->vmcb->save.rax = data & 0xffffffff;
  1024. svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;
  1025. svm->next_rip = svm->vmcb->save.rip + 2;
  1026. skip_emulated_instruction(&svm->vcpu);
  1027. }
  1028. return 1;
  1029. }
  1030. static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
  1031. {
  1032. struct vcpu_svm *svm = to_svm(vcpu);
  1033. switch (ecx) {
  1034. case MSR_IA32_TIME_STAMP_COUNTER: {
  1035. u64 tsc;
  1036. rdtscll(tsc);
  1037. svm->vmcb->control.tsc_offset = data - tsc;
  1038. break;
  1039. }
  1040. case MSR_K6_STAR:
  1041. svm->vmcb->save.star = data;
  1042. break;
  1043. #ifdef CONFIG_X86_64
  1044. case MSR_LSTAR:
  1045. svm->vmcb->save.lstar = data;
  1046. break;
  1047. case MSR_CSTAR:
  1048. svm->vmcb->save.cstar = data;
  1049. break;
  1050. case MSR_KERNEL_GS_BASE:
  1051. svm->vmcb->save.kernel_gs_base = data;
  1052. break;
  1053. case MSR_SYSCALL_MASK:
  1054. svm->vmcb->save.sfmask = data;
  1055. break;
  1056. #endif
  1057. case MSR_IA32_SYSENTER_CS:
  1058. svm->vmcb->save.sysenter_cs = data;
  1059. break;
  1060. case MSR_IA32_SYSENTER_EIP:
  1061. svm->vmcb->save.sysenter_eip = data;
  1062. break;
  1063. case MSR_IA32_SYSENTER_ESP:
  1064. svm->vmcb->save.sysenter_esp = data;
  1065. break;
  1066. case MSR_IA32_DEBUGCTLMSR:
  1067. if (!svm_has(SVM_FEATURE_LBRV)) {
  1068. pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
  1069. __func__, data);
  1070. break;
  1071. }
  1072. if (data & DEBUGCTL_RESERVED_BITS)
  1073. return 1;
  1074. svm->vmcb->save.dbgctl = data;
  1075. if (data & (1ULL<<0))
  1076. svm_enable_lbrv(svm);
  1077. else
  1078. svm_disable_lbrv(svm);
  1079. break;
  1080. case MSR_K7_EVNTSEL0:
  1081. case MSR_K7_EVNTSEL1:
  1082. case MSR_K7_EVNTSEL2:
  1083. case MSR_K7_EVNTSEL3:
  1084. /*
  1085. * only support writing 0 to the performance counters for now
  1086. * to make Windows happy. Should be replaced by a real
  1087. * performance counter emulation later.
  1088. */
  1089. if (data != 0)
  1090. goto unhandled;
  1091. break;
  1092. default:
  1093. unhandled:
  1094. return kvm_set_msr_common(vcpu, ecx, data);
  1095. }
  1096. return 0;
  1097. }
  1098. static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  1099. {
  1100. u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
  1101. u64 data = (svm->vmcb->save.rax & -1u)
  1102. | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
  1103. svm->next_rip = svm->vmcb->save.rip + 2;
  1104. if (svm_set_msr(&svm->vcpu, ecx, data))
  1105. kvm_inject_gp(&svm->vcpu, 0);
  1106. else
  1107. skip_emulated_instruction(&svm->vcpu);
  1108. return 1;
  1109. }
  1110. static int msr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  1111. {
  1112. if (svm->vmcb->control.exit_info_1)
  1113. return wrmsr_interception(svm, kvm_run);
  1114. else
  1115. return rdmsr_interception(svm, kvm_run);
  1116. }
  1117. static int interrupt_window_interception(struct vcpu_svm *svm,
  1118. struct kvm_run *kvm_run)
  1119. {
  1120. svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
  1121. svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
  1122. /*
  1123. * If the user space waits to inject interrupts, exit as soon as
  1124. * possible
  1125. */
  1126. if (kvm_run->request_interrupt_window &&
  1127. !svm->vcpu.arch.irq_summary) {
  1128. ++svm->vcpu.stat.irq_window_exits;
  1129. kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
  1130. return 0;
  1131. }
  1132. return 1;
  1133. }
  1134. static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
  1135. struct kvm_run *kvm_run) = {
  1136. [SVM_EXIT_READ_CR0] = emulate_on_interception,
  1137. [SVM_EXIT_READ_CR3] = emulate_on_interception,
  1138. [SVM_EXIT_READ_CR4] = emulate_on_interception,
  1139. [SVM_EXIT_READ_CR8] = emulate_on_interception,
  1140. /* for now: */
  1141. [SVM_EXIT_WRITE_CR0] = emulate_on_interception,
  1142. [SVM_EXIT_WRITE_CR3] = emulate_on_interception,
  1143. [SVM_EXIT_WRITE_CR4] = emulate_on_interception,
  1144. [SVM_EXIT_WRITE_CR8] = cr8_write_interception,
  1145. [SVM_EXIT_READ_DR0] = emulate_on_interception,
  1146. [SVM_EXIT_READ_DR1] = emulate_on_interception,
  1147. [SVM_EXIT_READ_DR2] = emulate_on_interception,
  1148. [SVM_EXIT_READ_DR3] = emulate_on_interception,
  1149. [SVM_EXIT_WRITE_DR0] = emulate_on_interception,
  1150. [SVM_EXIT_WRITE_DR1] = emulate_on_interception,
  1151. [SVM_EXIT_WRITE_DR2] = emulate_on_interception,
  1152. [SVM_EXIT_WRITE_DR3] = emulate_on_interception,
  1153. [SVM_EXIT_WRITE_DR5] = emulate_on_interception,
  1154. [SVM_EXIT_WRITE_DR7] = emulate_on_interception,
  1155. [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception,
  1156. [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception,
  1157. [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception,
  1158. [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception,
  1159. [SVM_EXIT_INTR] = nop_on_interception,
  1160. [SVM_EXIT_NMI] = nop_on_interception,
  1161. [SVM_EXIT_SMI] = nop_on_interception,
  1162. [SVM_EXIT_INIT] = nop_on_interception,
  1163. [SVM_EXIT_VINTR] = interrupt_window_interception,
  1164. /* [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception, */
  1165. [SVM_EXIT_CPUID] = cpuid_interception,
  1166. [SVM_EXIT_INVD] = emulate_on_interception,
  1167. [SVM_EXIT_HLT] = halt_interception,
  1168. [SVM_EXIT_INVLPG] = emulate_on_interception,
  1169. [SVM_EXIT_INVLPGA] = invalid_op_interception,
  1170. [SVM_EXIT_IOIO] = io_interception,
  1171. [SVM_EXIT_MSR] = msr_interception,
  1172. [SVM_EXIT_TASK_SWITCH] = task_switch_interception,
  1173. [SVM_EXIT_SHUTDOWN] = shutdown_interception,
  1174. [SVM_EXIT_VMRUN] = invalid_op_interception,
  1175. [SVM_EXIT_VMMCALL] = vmmcall_interception,
  1176. [SVM_EXIT_VMLOAD] = invalid_op_interception,
  1177. [SVM_EXIT_VMSAVE] = invalid_op_interception,
  1178. [SVM_EXIT_STGI] = invalid_op_interception,
  1179. [SVM_EXIT_CLGI] = invalid_op_interception,
  1180. [SVM_EXIT_SKINIT] = invalid_op_interception,
  1181. [SVM_EXIT_WBINVD] = emulate_on_interception,
  1182. [SVM_EXIT_MONITOR] = invalid_op_interception,
  1183. [SVM_EXIT_MWAIT] = invalid_op_interception,
  1184. [SVM_EXIT_NPF] = pf_interception,
  1185. };
  1186. static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
  1187. {
  1188. struct vcpu_svm *svm = to_svm(vcpu);
  1189. u32 exit_code = svm->vmcb->control.exit_code;
  1190. if (npt_enabled) {
  1191. int mmu_reload = 0;
  1192. if ((vcpu->arch.cr0 ^ svm->vmcb->save.cr0) & X86_CR0_PG) {
  1193. svm_set_cr0(vcpu, svm->vmcb->save.cr0);
  1194. mmu_reload = 1;
  1195. }
  1196. vcpu->arch.cr0 = svm->vmcb->save.cr0;
  1197. vcpu->arch.cr3 = svm->vmcb->save.cr3;
  1198. if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
  1199. if (!load_pdptrs(vcpu, vcpu->arch.cr3)) {
  1200. kvm_inject_gp(vcpu, 0);
  1201. return 1;
  1202. }
  1203. }
  1204. if (mmu_reload) {
  1205. kvm_mmu_reset_context(vcpu);
  1206. kvm_mmu_load(vcpu);
  1207. }
  1208. }
  1209. kvm_reput_irq(svm);
  1210. if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
  1211. kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
  1212. kvm_run->fail_entry.hardware_entry_failure_reason
  1213. = svm->vmcb->control.exit_code;
  1214. return 0;
  1215. }
  1216. if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
  1217. exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
  1218. exit_code != SVM_EXIT_NPF)
  1219. printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
  1220. "exit_code 0x%x\n",
  1221. __func__, svm->vmcb->control.exit_int_info,
  1222. exit_code);
  1223. if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
  1224. || !svm_exit_handlers[exit_code]) {
  1225. kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
  1226. kvm_run->hw.hardware_exit_reason = exit_code;
  1227. return 0;
  1228. }
  1229. return svm_exit_handlers[exit_code](svm, kvm_run);
  1230. }
  1231. static void reload_tss(struct kvm_vcpu *vcpu)
  1232. {
  1233. int cpu = raw_smp_processor_id();
  1234. struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
  1235. svm_data->tss_desc->type = 9; /* available 32/64-bit TSS */
  1236. load_TR_desc();
  1237. }
  1238. static void pre_svm_run(struct vcpu_svm *svm)
  1239. {
  1240. int cpu = raw_smp_processor_id();
  1241. struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
  1242. svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
  1243. if (svm->vcpu.cpu != cpu ||
  1244. svm->asid_generation != svm_data->asid_generation)
  1245. new_asid(svm, svm_data);
  1246. }
  1247. static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
  1248. {
  1249. struct vmcb_control_area *control;
  1250. control = &svm->vmcb->control;
  1251. control->int_vector = irq;
  1252. control->int_ctl &= ~V_INTR_PRIO_MASK;
  1253. control->int_ctl |= V_IRQ_MASK |
  1254. ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
  1255. }
  1256. static void svm_set_irq(struct kvm_vcpu *vcpu, int irq)
  1257. {
  1258. struct vcpu_svm *svm = to_svm(vcpu);
  1259. svm_inject_irq(svm, irq);
  1260. }
  1261. static void svm_intr_assist(struct kvm_vcpu *vcpu)
  1262. {
  1263. struct vcpu_svm *svm = to_svm(vcpu);
  1264. struct vmcb *vmcb = svm->vmcb;
  1265. int intr_vector = -1;
  1266. if ((vmcb->control.exit_int_info & SVM_EVTINJ_VALID) &&
  1267. ((vmcb->control.exit_int_info & SVM_EVTINJ_TYPE_MASK) == 0)) {
  1268. intr_vector = vmcb->control.exit_int_info &
  1269. SVM_EVTINJ_VEC_MASK;
  1270. vmcb->control.exit_int_info = 0;
  1271. svm_inject_irq(svm, intr_vector);
  1272. return;
  1273. }
  1274. if (vmcb->control.int_ctl & V_IRQ_MASK)
  1275. return;
  1276. if (!kvm_cpu_has_interrupt(vcpu))
  1277. return;
  1278. if (!(vmcb->save.rflags & X86_EFLAGS_IF) ||
  1279. (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) ||
  1280. (vmcb->control.event_inj & SVM_EVTINJ_VALID)) {
  1281. /* unable to deliver irq, set pending irq */
  1282. vmcb->control.intercept |= (1ULL << INTERCEPT_VINTR);
  1283. svm_inject_irq(svm, 0x0);
  1284. return;
  1285. }
  1286. /* Okay, we can deliver the interrupt: grab it and update PIC state. */
  1287. intr_vector = kvm_cpu_get_interrupt(vcpu);
  1288. svm_inject_irq(svm, intr_vector);
  1289. kvm_timer_intr_post(vcpu, intr_vector);
  1290. }
  1291. static void kvm_reput_irq(struct vcpu_svm *svm)
  1292. {
  1293. struct vmcb_control_area *control = &svm->vmcb->control;
  1294. if ((control->int_ctl & V_IRQ_MASK)
  1295. && !irqchip_in_kernel(svm->vcpu.kvm)) {
  1296. control->int_ctl &= ~V_IRQ_MASK;
  1297. push_irq(&svm->vcpu, control->int_vector);
  1298. }
  1299. svm->vcpu.arch.interrupt_window_open =
  1300. !(control->int_state & SVM_INTERRUPT_SHADOW_MASK);
  1301. }
  1302. static void svm_do_inject_vector(struct vcpu_svm *svm)
  1303. {
  1304. struct kvm_vcpu *vcpu = &svm->vcpu;
  1305. int word_index = __ffs(vcpu->arch.irq_summary);
  1306. int bit_index = __ffs(vcpu->arch.irq_pending[word_index]);
  1307. int irq = word_index * BITS_PER_LONG + bit_index;
  1308. clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]);
  1309. if (!vcpu->arch.irq_pending[word_index])
  1310. clear_bit(word_index, &vcpu->arch.irq_summary);
  1311. svm_inject_irq(svm, irq);
  1312. }
  1313. static void do_interrupt_requests(struct kvm_vcpu *vcpu,
  1314. struct kvm_run *kvm_run)
  1315. {
  1316. struct vcpu_svm *svm = to_svm(vcpu);
  1317. struct vmcb_control_area *control = &svm->vmcb->control;
  1318. svm->vcpu.arch.interrupt_window_open =
  1319. (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) &&
  1320. (svm->vmcb->save.rflags & X86_EFLAGS_IF));
  1321. if (svm->vcpu.arch.interrupt_window_open && svm->vcpu.arch.irq_summary)
  1322. /*
  1323. * If interrupts enabled, and not blocked by sti or mov ss. Good.
  1324. */
  1325. svm_do_inject_vector(svm);
  1326. /*
  1327. * Interrupts blocked. Wait for unblock.
  1328. */
  1329. if (!svm->vcpu.arch.interrupt_window_open &&
  1330. (svm->vcpu.arch.irq_summary || kvm_run->request_interrupt_window))
  1331. control->intercept |= 1ULL << INTERCEPT_VINTR;
  1332. else
  1333. control->intercept &= ~(1ULL << INTERCEPT_VINTR);
  1334. }
  1335. static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
  1336. {
  1337. return 0;
  1338. }
  1339. static void save_db_regs(unsigned long *db_regs)
  1340. {
  1341. asm volatile ("mov %%dr0, %0" : "=r"(db_regs[0]));
  1342. asm volatile ("mov %%dr1, %0" : "=r"(db_regs[1]));
  1343. asm volatile ("mov %%dr2, %0" : "=r"(db_regs[2]));
  1344. asm volatile ("mov %%dr3, %0" : "=r"(db_regs[3]));
  1345. }
  1346. static void load_db_regs(unsigned long *db_regs)
  1347. {
  1348. asm volatile ("mov %0, %%dr0" : : "r"(db_regs[0]));
  1349. asm volatile ("mov %0, %%dr1" : : "r"(db_regs[1]));
  1350. asm volatile ("mov %0, %%dr2" : : "r"(db_regs[2]));
  1351. asm volatile ("mov %0, %%dr3" : : "r"(db_regs[3]));
  1352. }
  1353. static void svm_flush_tlb(struct kvm_vcpu *vcpu)
  1354. {
  1355. force_new_asid(vcpu);
  1356. }
  1357. static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
  1358. {
  1359. }
  1360. static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1361. {
  1362. struct vcpu_svm *svm = to_svm(vcpu);
  1363. u16 fs_selector;
  1364. u16 gs_selector;
  1365. u16 ldt_selector;
  1366. pre_svm_run(svm);
  1367. save_host_msrs(vcpu);
  1368. fs_selector = read_fs();
  1369. gs_selector = read_gs();
  1370. ldt_selector = read_ldt();
  1371. svm->host_cr2 = kvm_read_cr2();
  1372. svm->host_dr6 = read_dr6();
  1373. svm->host_dr7 = read_dr7();
  1374. svm->vmcb->save.cr2 = vcpu->arch.cr2;
  1375. /* required for live migration with NPT */
  1376. if (npt_enabled)
  1377. svm->vmcb->save.cr3 = vcpu->arch.cr3;
  1378. if (svm->vmcb->save.dr7 & 0xff) {
  1379. write_dr7(0);
  1380. save_db_regs(svm->host_db_regs);
  1381. load_db_regs(svm->db_regs);
  1382. }
  1383. clgi();
  1384. local_irq_enable();
  1385. asm volatile (
  1386. #ifdef CONFIG_X86_64
  1387. "push %%rbp; \n\t"
  1388. #else
  1389. "push %%ebp; \n\t"
  1390. #endif
  1391. #ifdef CONFIG_X86_64
  1392. "mov %c[rbx](%[svm]), %%rbx \n\t"
  1393. "mov %c[rcx](%[svm]), %%rcx \n\t"
  1394. "mov %c[rdx](%[svm]), %%rdx \n\t"
  1395. "mov %c[rsi](%[svm]), %%rsi \n\t"
  1396. "mov %c[rdi](%[svm]), %%rdi \n\t"
  1397. "mov %c[rbp](%[svm]), %%rbp \n\t"
  1398. "mov %c[r8](%[svm]), %%r8 \n\t"
  1399. "mov %c[r9](%[svm]), %%r9 \n\t"
  1400. "mov %c[r10](%[svm]), %%r10 \n\t"
  1401. "mov %c[r11](%[svm]), %%r11 \n\t"
  1402. "mov %c[r12](%[svm]), %%r12 \n\t"
  1403. "mov %c[r13](%[svm]), %%r13 \n\t"
  1404. "mov %c[r14](%[svm]), %%r14 \n\t"
  1405. "mov %c[r15](%[svm]), %%r15 \n\t"
  1406. #else
  1407. "mov %c[rbx](%[svm]), %%ebx \n\t"
  1408. "mov %c[rcx](%[svm]), %%ecx \n\t"
  1409. "mov %c[rdx](%[svm]), %%edx \n\t"
  1410. "mov %c[rsi](%[svm]), %%esi \n\t"
  1411. "mov %c[rdi](%[svm]), %%edi \n\t"
  1412. "mov %c[rbp](%[svm]), %%ebp \n\t"
  1413. #endif
  1414. #ifdef CONFIG_X86_64
  1415. /* Enter guest mode */
  1416. "push %%rax \n\t"
  1417. "mov %c[vmcb](%[svm]), %%rax \n\t"
  1418. SVM_VMLOAD "\n\t"
  1419. SVM_VMRUN "\n\t"
  1420. SVM_VMSAVE "\n\t"
  1421. "pop %%rax \n\t"
  1422. #else
  1423. /* Enter guest mode */
  1424. "push %%eax \n\t"
  1425. "mov %c[vmcb](%[svm]), %%eax \n\t"
  1426. SVM_VMLOAD "\n\t"
  1427. SVM_VMRUN "\n\t"
  1428. SVM_VMSAVE "\n\t"
  1429. "pop %%eax \n\t"
  1430. #endif
  1431. /* Save guest registers, load host registers */
  1432. #ifdef CONFIG_X86_64
  1433. "mov %%rbx, %c[rbx](%[svm]) \n\t"
  1434. "mov %%rcx, %c[rcx](%[svm]) \n\t"
  1435. "mov %%rdx, %c[rdx](%[svm]) \n\t"
  1436. "mov %%rsi, %c[rsi](%[svm]) \n\t"
  1437. "mov %%rdi, %c[rdi](%[svm]) \n\t"
  1438. "mov %%rbp, %c[rbp](%[svm]) \n\t"
  1439. "mov %%r8, %c[r8](%[svm]) \n\t"
  1440. "mov %%r9, %c[r9](%[svm]) \n\t"
  1441. "mov %%r10, %c[r10](%[svm]) \n\t"
  1442. "mov %%r11, %c[r11](%[svm]) \n\t"
  1443. "mov %%r12, %c[r12](%[svm]) \n\t"
  1444. "mov %%r13, %c[r13](%[svm]) \n\t"
  1445. "mov %%r14, %c[r14](%[svm]) \n\t"
  1446. "mov %%r15, %c[r15](%[svm]) \n\t"
  1447. "pop %%rbp; \n\t"
  1448. #else
  1449. "mov %%ebx, %c[rbx](%[svm]) \n\t"
  1450. "mov %%ecx, %c[rcx](%[svm]) \n\t"
  1451. "mov %%edx, %c[rdx](%[svm]) \n\t"
  1452. "mov %%esi, %c[rsi](%[svm]) \n\t"
  1453. "mov %%edi, %c[rdi](%[svm]) \n\t"
  1454. "mov %%ebp, %c[rbp](%[svm]) \n\t"
  1455. "pop %%ebp; \n\t"
  1456. #endif
  1457. :
  1458. : [svm]"a"(svm),
  1459. [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
  1460. [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
  1461. [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
  1462. [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
  1463. [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
  1464. [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
  1465. [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
  1466. #ifdef CONFIG_X86_64
  1467. , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
  1468. [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
  1469. [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
  1470. [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
  1471. [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
  1472. [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
  1473. [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
  1474. [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
  1475. #endif
  1476. : "cc", "memory"
  1477. #ifdef CONFIG_X86_64
  1478. , "rbx", "rcx", "rdx", "rsi", "rdi"
  1479. , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
  1480. #else
  1481. , "ebx", "ecx", "edx" , "esi", "edi"
  1482. #endif
  1483. );
  1484. if ((svm->vmcb->save.dr7 & 0xff))
  1485. load_db_regs(svm->host_db_regs);
  1486. vcpu->arch.cr2 = svm->vmcb->save.cr2;
  1487. write_dr6(svm->host_dr6);
  1488. write_dr7(svm->host_dr7);
  1489. kvm_write_cr2(svm->host_cr2);
  1490. load_fs(fs_selector);
  1491. load_gs(gs_selector);
  1492. load_ldt(ldt_selector);
  1493. load_host_msrs(vcpu);
  1494. reload_tss(vcpu);
  1495. local_irq_disable();
  1496. stgi();
  1497. svm->next_rip = 0;
  1498. }
  1499. static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
  1500. {
  1501. struct vcpu_svm *svm = to_svm(vcpu);
  1502. if (npt_enabled) {
  1503. svm->vmcb->control.nested_cr3 = root;
  1504. force_new_asid(vcpu);
  1505. return;
  1506. }
  1507. svm->vmcb->save.cr3 = root;
  1508. force_new_asid(vcpu);
  1509. if (vcpu->fpu_active) {
  1510. svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR);
  1511. svm->vmcb->save.cr0 |= X86_CR0_TS;
  1512. vcpu->fpu_active = 0;
  1513. }
  1514. }
  1515. static int is_disabled(void)
  1516. {
  1517. u64 vm_cr;
  1518. rdmsrl(MSR_VM_CR, vm_cr);
  1519. if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
  1520. return 1;
  1521. return 0;
  1522. }
  1523. static void
  1524. svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
  1525. {
  1526. /*
  1527. * Patch in the VMMCALL instruction:
  1528. */
  1529. hypercall[0] = 0x0f;
  1530. hypercall[1] = 0x01;
  1531. hypercall[2] = 0xd9;
  1532. }
  1533. static void svm_check_processor_compat(void *rtn)
  1534. {
  1535. *(int *)rtn = 0;
  1536. }
  1537. static bool svm_cpu_has_accelerated_tpr(void)
  1538. {
  1539. return false;
  1540. }
  1541. static struct kvm_x86_ops svm_x86_ops = {
  1542. .cpu_has_kvm_support = has_svm,
  1543. .disabled_by_bios = is_disabled,
  1544. .hardware_setup = svm_hardware_setup,
  1545. .hardware_unsetup = svm_hardware_unsetup,
  1546. .check_processor_compatibility = svm_check_processor_compat,
  1547. .hardware_enable = svm_hardware_enable,
  1548. .hardware_disable = svm_hardware_disable,
  1549. .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
  1550. .vcpu_create = svm_create_vcpu,
  1551. .vcpu_free = svm_free_vcpu,
  1552. .vcpu_reset = svm_vcpu_reset,
  1553. .prepare_guest_switch = svm_prepare_guest_switch,
  1554. .vcpu_load = svm_vcpu_load,
  1555. .vcpu_put = svm_vcpu_put,
  1556. .vcpu_decache = svm_vcpu_decache,
  1557. .set_guest_debug = svm_guest_debug,
  1558. .get_msr = svm_get_msr,
  1559. .set_msr = svm_set_msr,
  1560. .get_segment_base = svm_get_segment_base,
  1561. .get_segment = svm_get_segment,
  1562. .set_segment = svm_set_segment,
  1563. .get_cpl = svm_get_cpl,
  1564. .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
  1565. .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
  1566. .set_cr0 = svm_set_cr0,
  1567. .set_cr3 = svm_set_cr3,
  1568. .set_cr4 = svm_set_cr4,
  1569. .set_efer = svm_set_efer,
  1570. .get_idt = svm_get_idt,
  1571. .set_idt = svm_set_idt,
  1572. .get_gdt = svm_get_gdt,
  1573. .set_gdt = svm_set_gdt,
  1574. .get_dr = svm_get_dr,
  1575. .set_dr = svm_set_dr,
  1576. .cache_regs = svm_cache_regs,
  1577. .decache_regs = svm_decache_regs,
  1578. .get_rflags = svm_get_rflags,
  1579. .set_rflags = svm_set_rflags,
  1580. .tlb_flush = svm_flush_tlb,
  1581. .run = svm_vcpu_run,
  1582. .handle_exit = handle_exit,
  1583. .skip_emulated_instruction = skip_emulated_instruction,
  1584. .patch_hypercall = svm_patch_hypercall,
  1585. .get_irq = svm_get_irq,
  1586. .set_irq = svm_set_irq,
  1587. .queue_exception = svm_queue_exception,
  1588. .exception_injected = svm_exception_injected,
  1589. .inject_pending_irq = svm_intr_assist,
  1590. .inject_pending_vectors = do_interrupt_requests,
  1591. .set_tss_addr = svm_set_tss_addr,
  1592. };
  1593. static int __init svm_init(void)
  1594. {
  1595. return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
  1596. THIS_MODULE);
  1597. }
  1598. static void __exit svm_exit(void)
  1599. {
  1600. kvm_exit();
  1601. }
  1602. module_init(svm_init)
  1603. module_exit(svm_exit)