svm.c 43 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * AMD SVM support
  5. *
  6. * Copyright (C) 2006 Qumranet, Inc.
  7. *
  8. * Authors:
  9. * Yaniv Kamay <yaniv@qumranet.com>
  10. * Avi Kivity <avi@qumranet.com>
  11. *
  12. * This work is licensed under the terms of the GNU GPL, version 2. See
  13. * the COPYING file in the top-level directory.
  14. *
  15. */
  16. #include "kvm_svm.h"
  17. #include "x86_emulate.h"
  18. #include "irq.h"
  19. #include <linux/module.h>
  20. #include <linux/kernel.h>
  21. #include <linux/vmalloc.h>
  22. #include <linux/highmem.h>
  23. #include <linux/sched.h>
  24. #include <asm/desc.h>
  25. MODULE_AUTHOR("Qumranet");
  26. MODULE_LICENSE("GPL");
  27. #define IOPM_ALLOC_ORDER 2
  28. #define MSRPM_ALLOC_ORDER 1
  29. #define DB_VECTOR 1
  30. #define UD_VECTOR 6
  31. #define GP_VECTOR 13
  32. #define DR7_GD_MASK (1 << 13)
  33. #define DR6_BD_MASK (1 << 13)
  34. #define SEG_TYPE_LDT 2
  35. #define SEG_TYPE_BUSY_TSS16 3
  36. #define KVM_EFER_LMA (1 << 10)
  37. #define KVM_EFER_LME (1 << 8)
  38. #define SVM_FEATURE_NPT (1 << 0)
  39. #define SVM_FEATURE_LBRV (1 << 1)
  40. #define SVM_DEATURE_SVML (1 << 2)
  41. static void kvm_reput_irq(struct vcpu_svm *svm);
  42. static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
  43. {
  44. return container_of(vcpu, struct vcpu_svm, vcpu);
  45. }
  46. unsigned long iopm_base;
  47. unsigned long msrpm_base;
  48. struct kvm_ldttss_desc {
  49. u16 limit0;
  50. u16 base0;
  51. unsigned base1 : 8, type : 5, dpl : 2, p : 1;
  52. unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8;
  53. u32 base3;
  54. u32 zero1;
  55. } __attribute__((packed));
  56. struct svm_cpu_data {
  57. int cpu;
  58. u64 asid_generation;
  59. u32 max_asid;
  60. u32 next_asid;
  61. struct kvm_ldttss_desc *tss_desc;
  62. struct page *save_area;
  63. };
  64. static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
  65. static uint32_t svm_features;
  66. struct svm_init_data {
  67. int cpu;
  68. int r;
  69. };
  70. static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
  71. #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
  72. #define MSRS_RANGE_SIZE 2048
  73. #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
  74. #define MAX_INST_SIZE 15
  75. static inline u32 svm_has(u32 feat)
  76. {
  77. return svm_features & feat;
  78. }
  79. static inline u8 pop_irq(struct kvm_vcpu *vcpu)
  80. {
  81. int word_index = __ffs(vcpu->irq_summary);
  82. int bit_index = __ffs(vcpu->irq_pending[word_index]);
  83. int irq = word_index * BITS_PER_LONG + bit_index;
  84. clear_bit(bit_index, &vcpu->irq_pending[word_index]);
  85. if (!vcpu->irq_pending[word_index])
  86. clear_bit(word_index, &vcpu->irq_summary);
  87. return irq;
  88. }
  89. static inline void push_irq(struct kvm_vcpu *vcpu, u8 irq)
  90. {
  91. set_bit(irq, vcpu->irq_pending);
  92. set_bit(irq / BITS_PER_LONG, &vcpu->irq_summary);
  93. }
  94. static inline void clgi(void)
  95. {
  96. asm volatile (SVM_CLGI);
  97. }
  98. static inline void stgi(void)
  99. {
  100. asm volatile (SVM_STGI);
  101. }
  102. static inline void invlpga(unsigned long addr, u32 asid)
  103. {
  104. asm volatile (SVM_INVLPGA :: "a"(addr), "c"(asid));
  105. }
  106. static inline unsigned long kvm_read_cr2(void)
  107. {
  108. unsigned long cr2;
  109. asm volatile ("mov %%cr2, %0" : "=r" (cr2));
  110. return cr2;
  111. }
  112. static inline void kvm_write_cr2(unsigned long val)
  113. {
  114. asm volatile ("mov %0, %%cr2" :: "r" (val));
  115. }
  116. static inline unsigned long read_dr6(void)
  117. {
  118. unsigned long dr6;
  119. asm volatile ("mov %%dr6, %0" : "=r" (dr6));
  120. return dr6;
  121. }
  122. static inline void write_dr6(unsigned long val)
  123. {
  124. asm volatile ("mov %0, %%dr6" :: "r" (val));
  125. }
  126. static inline unsigned long read_dr7(void)
  127. {
  128. unsigned long dr7;
  129. asm volatile ("mov %%dr7, %0" : "=r" (dr7));
  130. return dr7;
  131. }
  132. static inline void write_dr7(unsigned long val)
  133. {
  134. asm volatile ("mov %0, %%dr7" :: "r" (val));
  135. }
  136. static inline void force_new_asid(struct kvm_vcpu *vcpu)
  137. {
  138. to_svm(vcpu)->asid_generation--;
  139. }
  140. static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
  141. {
  142. force_new_asid(vcpu);
  143. }
  144. static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
  145. {
  146. if (!(efer & KVM_EFER_LMA))
  147. efer &= ~KVM_EFER_LME;
  148. to_svm(vcpu)->vmcb->save.efer = efer | MSR_EFER_SVME_MASK;
  149. vcpu->shadow_efer = efer;
  150. }
  151. static void svm_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code)
  152. {
  153. struct vcpu_svm *svm = to_svm(vcpu);
  154. svm->vmcb->control.event_inj = SVM_EVTINJ_VALID |
  155. SVM_EVTINJ_VALID_ERR |
  156. SVM_EVTINJ_TYPE_EXEPT |
  157. GP_VECTOR;
  158. svm->vmcb->control.event_inj_err = error_code;
  159. }
  160. static void inject_ud(struct kvm_vcpu *vcpu)
  161. {
  162. to_svm(vcpu)->vmcb->control.event_inj = SVM_EVTINJ_VALID |
  163. SVM_EVTINJ_TYPE_EXEPT |
  164. UD_VECTOR;
  165. }
  166. static int is_page_fault(uint32_t info)
  167. {
  168. info &= SVM_EVTINJ_VEC_MASK | SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
  169. return info == (PF_VECTOR | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT);
  170. }
  171. static int is_external_interrupt(u32 info)
  172. {
  173. info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
  174. return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
  175. }
  176. static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
  177. {
  178. struct vcpu_svm *svm = to_svm(vcpu);
  179. if (!svm->next_rip) {
  180. printk(KERN_DEBUG "%s: NOP\n", __FUNCTION__);
  181. return;
  182. }
  183. if (svm->next_rip - svm->vmcb->save.rip > MAX_INST_SIZE) {
  184. printk(KERN_ERR "%s: ip 0x%llx next 0x%llx\n",
  185. __FUNCTION__,
  186. svm->vmcb->save.rip,
  187. svm->next_rip);
  188. }
  189. vcpu->rip = svm->vmcb->save.rip = svm->next_rip;
  190. svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
  191. vcpu->interrupt_window_open = 1;
  192. }
  193. static int has_svm(void)
  194. {
  195. uint32_t eax, ebx, ecx, edx;
  196. if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
  197. printk(KERN_INFO "has_svm: not amd\n");
  198. return 0;
  199. }
  200. cpuid(0x80000000, &eax, &ebx, &ecx, &edx);
  201. if (eax < SVM_CPUID_FUNC) {
  202. printk(KERN_INFO "has_svm: can't execute cpuid_8000000a\n");
  203. return 0;
  204. }
  205. cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
  206. if (!(ecx & (1 << SVM_CPUID_FEATURE_SHIFT))) {
  207. printk(KERN_DEBUG "has_svm: svm not available\n");
  208. return 0;
  209. }
  210. return 1;
  211. }
  212. static void svm_hardware_disable(void *garbage)
  213. {
  214. struct svm_cpu_data *svm_data
  215. = per_cpu(svm_data, raw_smp_processor_id());
  216. if (svm_data) {
  217. uint64_t efer;
  218. wrmsrl(MSR_VM_HSAVE_PA, 0);
  219. rdmsrl(MSR_EFER, efer);
  220. wrmsrl(MSR_EFER, efer & ~MSR_EFER_SVME_MASK);
  221. per_cpu(svm_data, raw_smp_processor_id()) = NULL;
  222. __free_page(svm_data->save_area);
  223. kfree(svm_data);
  224. }
  225. }
  226. static void svm_hardware_enable(void *garbage)
  227. {
  228. struct svm_cpu_data *svm_data;
  229. uint64_t efer;
  230. #ifdef CONFIG_X86_64
  231. struct desc_ptr gdt_descr;
  232. #else
  233. struct Xgt_desc_struct gdt_descr;
  234. #endif
  235. struct desc_struct *gdt;
  236. int me = raw_smp_processor_id();
  237. if (!has_svm()) {
  238. printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me);
  239. return;
  240. }
  241. svm_data = per_cpu(svm_data, me);
  242. if (!svm_data) {
  243. printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n",
  244. me);
  245. return;
  246. }
  247. svm_data->asid_generation = 1;
  248. svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
  249. svm_data->next_asid = svm_data->max_asid + 1;
  250. svm_features = cpuid_edx(SVM_CPUID_FUNC);
  251. asm volatile ( "sgdt %0" : "=m"(gdt_descr) );
  252. gdt = (struct desc_struct *)gdt_descr.address;
  253. svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
  254. rdmsrl(MSR_EFER, efer);
  255. wrmsrl(MSR_EFER, efer | MSR_EFER_SVME_MASK);
  256. wrmsrl(MSR_VM_HSAVE_PA,
  257. page_to_pfn(svm_data->save_area) << PAGE_SHIFT);
  258. }
  259. static int svm_cpu_init(int cpu)
  260. {
  261. struct svm_cpu_data *svm_data;
  262. int r;
  263. svm_data = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
  264. if (!svm_data)
  265. return -ENOMEM;
  266. svm_data->cpu = cpu;
  267. svm_data->save_area = alloc_page(GFP_KERNEL);
  268. r = -ENOMEM;
  269. if (!svm_data->save_area)
  270. goto err_1;
  271. per_cpu(svm_data, cpu) = svm_data;
  272. return 0;
  273. err_1:
  274. kfree(svm_data);
  275. return r;
  276. }
  277. static void set_msr_interception(u32 *msrpm, unsigned msr,
  278. int read, int write)
  279. {
  280. int i;
  281. for (i = 0; i < NUM_MSR_MAPS; i++) {
  282. if (msr >= msrpm_ranges[i] &&
  283. msr < msrpm_ranges[i] + MSRS_IN_RANGE) {
  284. u32 msr_offset = (i * MSRS_IN_RANGE + msr -
  285. msrpm_ranges[i]) * 2;
  286. u32 *base = msrpm + (msr_offset / 32);
  287. u32 msr_shift = msr_offset % 32;
  288. u32 mask = ((write) ? 0 : 2) | ((read) ? 0 : 1);
  289. *base = (*base & ~(0x3 << msr_shift)) |
  290. (mask << msr_shift);
  291. return;
  292. }
  293. }
  294. BUG();
  295. }
  296. static __init int svm_hardware_setup(void)
  297. {
  298. int cpu;
  299. struct page *iopm_pages;
  300. struct page *msrpm_pages;
  301. void *iopm_va, *msrpm_va;
  302. int r;
  303. iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
  304. if (!iopm_pages)
  305. return -ENOMEM;
  306. iopm_va = page_address(iopm_pages);
  307. memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
  308. clear_bit(0x80, iopm_va); /* allow direct access to PC debug port */
  309. iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
  310. msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
  311. r = -ENOMEM;
  312. if (!msrpm_pages)
  313. goto err_1;
  314. msrpm_va = page_address(msrpm_pages);
  315. memset(msrpm_va, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
  316. msrpm_base = page_to_pfn(msrpm_pages) << PAGE_SHIFT;
  317. #ifdef CONFIG_X86_64
  318. set_msr_interception(msrpm_va, MSR_GS_BASE, 1, 1);
  319. set_msr_interception(msrpm_va, MSR_FS_BASE, 1, 1);
  320. set_msr_interception(msrpm_va, MSR_KERNEL_GS_BASE, 1, 1);
  321. set_msr_interception(msrpm_va, MSR_LSTAR, 1, 1);
  322. set_msr_interception(msrpm_va, MSR_CSTAR, 1, 1);
  323. set_msr_interception(msrpm_va, MSR_SYSCALL_MASK, 1, 1);
  324. #endif
  325. set_msr_interception(msrpm_va, MSR_K6_STAR, 1, 1);
  326. set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_CS, 1, 1);
  327. set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_ESP, 1, 1);
  328. set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_EIP, 1, 1);
  329. for_each_online_cpu(cpu) {
  330. r = svm_cpu_init(cpu);
  331. if (r)
  332. goto err_2;
  333. }
  334. return 0;
  335. err_2:
  336. __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
  337. msrpm_base = 0;
  338. err_1:
  339. __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
  340. iopm_base = 0;
  341. return r;
  342. }
  343. static __exit void svm_hardware_unsetup(void)
  344. {
  345. __free_pages(pfn_to_page(msrpm_base >> PAGE_SHIFT), MSRPM_ALLOC_ORDER);
  346. __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
  347. iopm_base = msrpm_base = 0;
  348. }
  349. static void init_seg(struct vmcb_seg *seg)
  350. {
  351. seg->selector = 0;
  352. seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
  353. SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
  354. seg->limit = 0xffff;
  355. seg->base = 0;
  356. }
  357. static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
  358. {
  359. seg->selector = 0;
  360. seg->attrib = SVM_SELECTOR_P_MASK | type;
  361. seg->limit = 0xffff;
  362. seg->base = 0;
  363. }
  364. static void init_vmcb(struct vmcb *vmcb)
  365. {
  366. struct vmcb_control_area *control = &vmcb->control;
  367. struct vmcb_save_area *save = &vmcb->save;
  368. control->intercept_cr_read = INTERCEPT_CR0_MASK |
  369. INTERCEPT_CR3_MASK |
  370. INTERCEPT_CR4_MASK;
  371. control->intercept_cr_write = INTERCEPT_CR0_MASK |
  372. INTERCEPT_CR3_MASK |
  373. INTERCEPT_CR4_MASK;
  374. control->intercept_dr_read = INTERCEPT_DR0_MASK |
  375. INTERCEPT_DR1_MASK |
  376. INTERCEPT_DR2_MASK |
  377. INTERCEPT_DR3_MASK;
  378. control->intercept_dr_write = INTERCEPT_DR0_MASK |
  379. INTERCEPT_DR1_MASK |
  380. INTERCEPT_DR2_MASK |
  381. INTERCEPT_DR3_MASK |
  382. INTERCEPT_DR5_MASK |
  383. INTERCEPT_DR7_MASK;
  384. control->intercept_exceptions = 1 << PF_VECTOR;
  385. control->intercept = (1ULL << INTERCEPT_INTR) |
  386. (1ULL << INTERCEPT_NMI) |
  387. (1ULL << INTERCEPT_SMI) |
  388. /*
  389. * selective cr0 intercept bug?
  390. * 0: 0f 22 d8 mov %eax,%cr3
  391. * 3: 0f 20 c0 mov %cr0,%eax
  392. * 6: 0d 00 00 00 80 or $0x80000000,%eax
  393. * b: 0f 22 c0 mov %eax,%cr0
  394. * set cr3 ->interception
  395. * get cr0 ->interception
  396. * set cr0 -> no interception
  397. */
  398. /* (1ULL << INTERCEPT_SELECTIVE_CR0) | */
  399. (1ULL << INTERCEPT_CPUID) |
  400. (1ULL << INTERCEPT_INVD) |
  401. (1ULL << INTERCEPT_HLT) |
  402. (1ULL << INTERCEPT_INVLPGA) |
  403. (1ULL << INTERCEPT_IOIO_PROT) |
  404. (1ULL << INTERCEPT_MSR_PROT) |
  405. (1ULL << INTERCEPT_TASK_SWITCH) |
  406. (1ULL << INTERCEPT_SHUTDOWN) |
  407. (1ULL << INTERCEPT_VMRUN) |
  408. (1ULL << INTERCEPT_VMMCALL) |
  409. (1ULL << INTERCEPT_VMLOAD) |
  410. (1ULL << INTERCEPT_VMSAVE) |
  411. (1ULL << INTERCEPT_STGI) |
  412. (1ULL << INTERCEPT_CLGI) |
  413. (1ULL << INTERCEPT_SKINIT) |
  414. (1ULL << INTERCEPT_WBINVD) |
  415. (1ULL << INTERCEPT_MONITOR) |
  416. (1ULL << INTERCEPT_MWAIT);
  417. control->iopm_base_pa = iopm_base;
  418. control->msrpm_base_pa = msrpm_base;
  419. control->tsc_offset = 0;
  420. control->int_ctl = V_INTR_MASKING_MASK;
  421. init_seg(&save->es);
  422. init_seg(&save->ss);
  423. init_seg(&save->ds);
  424. init_seg(&save->fs);
  425. init_seg(&save->gs);
  426. save->cs.selector = 0xf000;
  427. /* Executable/Readable Code Segment */
  428. save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
  429. SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
  430. save->cs.limit = 0xffff;
  431. /*
  432. * cs.base should really be 0xffff0000, but vmx can't handle that, so
  433. * be consistent with it.
  434. *
  435. * Replace when we have real mode working for vmx.
  436. */
  437. save->cs.base = 0xf0000;
  438. save->gdtr.limit = 0xffff;
  439. save->idtr.limit = 0xffff;
  440. init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
  441. init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
  442. save->efer = MSR_EFER_SVME_MASK;
  443. save->dr6 = 0xffff0ff0;
  444. save->dr7 = 0x400;
  445. save->rflags = 2;
  446. save->rip = 0x0000fff0;
  447. /*
  448. * cr0 val on cpu init should be 0x60000010, we enable cpu
  449. * cache by default. the orderly way is to enable cache in bios.
  450. */
  451. save->cr0 = 0x00000010 | X86_CR0_PG | X86_CR0_WP;
  452. save->cr4 = X86_CR4_PAE;
  453. /* rdx = ?? */
  454. }
  455. static void svm_vcpu_reset(struct kvm_vcpu *vcpu)
  456. {
  457. struct vcpu_svm *svm = to_svm(vcpu);
  458. init_vmcb(svm->vmcb);
  459. if (vcpu->vcpu_id != 0) {
  460. svm->vmcb->save.rip = 0;
  461. svm->vmcb->save.cs.base = svm->vcpu.sipi_vector << 12;
  462. svm->vmcb->save.cs.selector = svm->vcpu.sipi_vector << 8;
  463. }
  464. }
  465. static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
  466. {
  467. struct vcpu_svm *svm;
  468. struct page *page;
  469. int err;
  470. svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
  471. if (!svm) {
  472. err = -ENOMEM;
  473. goto out;
  474. }
  475. err = kvm_vcpu_init(&svm->vcpu, kvm, id);
  476. if (err)
  477. goto free_svm;
  478. if (irqchip_in_kernel(kvm)) {
  479. err = kvm_create_lapic(&svm->vcpu);
  480. if (err < 0)
  481. goto free_svm;
  482. }
  483. page = alloc_page(GFP_KERNEL);
  484. if (!page) {
  485. err = -ENOMEM;
  486. goto uninit;
  487. }
  488. svm->vmcb = page_address(page);
  489. clear_page(svm->vmcb);
  490. svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
  491. svm->asid_generation = 0;
  492. memset(svm->db_regs, 0, sizeof(svm->db_regs));
  493. init_vmcb(svm->vmcb);
  494. fx_init(&svm->vcpu);
  495. svm->vcpu.fpu_active = 1;
  496. svm->vcpu.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
  497. if (svm->vcpu.vcpu_id == 0)
  498. svm->vcpu.apic_base |= MSR_IA32_APICBASE_BSP;
  499. return &svm->vcpu;
  500. uninit:
  501. kvm_vcpu_uninit(&svm->vcpu);
  502. free_svm:
  503. kmem_cache_free(kvm_vcpu_cache, svm);
  504. out:
  505. return ERR_PTR(err);
  506. }
  507. static void svm_free_vcpu(struct kvm_vcpu *vcpu)
  508. {
  509. struct vcpu_svm *svm = to_svm(vcpu);
  510. __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
  511. kvm_vcpu_uninit(vcpu);
  512. kmem_cache_free(kvm_vcpu_cache, svm);
  513. }
  514. static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  515. {
  516. struct vcpu_svm *svm = to_svm(vcpu);
  517. int i;
  518. if (unlikely(cpu != vcpu->cpu)) {
  519. u64 tsc_this, delta;
  520. /*
  521. * Make sure that the guest sees a monotonically
  522. * increasing TSC.
  523. */
  524. rdtscll(tsc_this);
  525. delta = vcpu->host_tsc - tsc_this;
  526. svm->vmcb->control.tsc_offset += delta;
  527. vcpu->cpu = cpu;
  528. kvm_migrate_apic_timer(vcpu);
  529. }
  530. for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
  531. rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
  532. }
  533. static void svm_vcpu_put(struct kvm_vcpu *vcpu)
  534. {
  535. struct vcpu_svm *svm = to_svm(vcpu);
  536. int i;
  537. for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
  538. wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
  539. rdtscll(vcpu->host_tsc);
  540. }
  541. static void svm_vcpu_decache(struct kvm_vcpu *vcpu)
  542. {
  543. }
  544. static void svm_cache_regs(struct kvm_vcpu *vcpu)
  545. {
  546. struct vcpu_svm *svm = to_svm(vcpu);
  547. vcpu->regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
  548. vcpu->regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
  549. vcpu->rip = svm->vmcb->save.rip;
  550. }
  551. static void svm_decache_regs(struct kvm_vcpu *vcpu)
  552. {
  553. struct vcpu_svm *svm = to_svm(vcpu);
  554. svm->vmcb->save.rax = vcpu->regs[VCPU_REGS_RAX];
  555. svm->vmcb->save.rsp = vcpu->regs[VCPU_REGS_RSP];
  556. svm->vmcb->save.rip = vcpu->rip;
  557. }
  558. static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
  559. {
  560. return to_svm(vcpu)->vmcb->save.rflags;
  561. }
  562. static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  563. {
  564. to_svm(vcpu)->vmcb->save.rflags = rflags;
  565. }
  566. static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
  567. {
  568. struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
  569. switch (seg) {
  570. case VCPU_SREG_CS: return &save->cs;
  571. case VCPU_SREG_DS: return &save->ds;
  572. case VCPU_SREG_ES: return &save->es;
  573. case VCPU_SREG_FS: return &save->fs;
  574. case VCPU_SREG_GS: return &save->gs;
  575. case VCPU_SREG_SS: return &save->ss;
  576. case VCPU_SREG_TR: return &save->tr;
  577. case VCPU_SREG_LDTR: return &save->ldtr;
  578. }
  579. BUG();
  580. return NULL;
  581. }
  582. static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
  583. {
  584. struct vmcb_seg *s = svm_seg(vcpu, seg);
  585. return s->base;
  586. }
  587. static void svm_get_segment(struct kvm_vcpu *vcpu,
  588. struct kvm_segment *var, int seg)
  589. {
  590. struct vmcb_seg *s = svm_seg(vcpu, seg);
  591. var->base = s->base;
  592. var->limit = s->limit;
  593. var->selector = s->selector;
  594. var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
  595. var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
  596. var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
  597. var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
  598. var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
  599. var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
  600. var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
  601. var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1;
  602. var->unusable = !var->present;
  603. }
  604. static void svm_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  605. {
  606. struct vcpu_svm *svm = to_svm(vcpu);
  607. dt->limit = svm->vmcb->save.idtr.limit;
  608. dt->base = svm->vmcb->save.idtr.base;
  609. }
  610. static void svm_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  611. {
  612. struct vcpu_svm *svm = to_svm(vcpu);
  613. svm->vmcb->save.idtr.limit = dt->limit;
  614. svm->vmcb->save.idtr.base = dt->base ;
  615. }
  616. static void svm_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  617. {
  618. struct vcpu_svm *svm = to_svm(vcpu);
  619. dt->limit = svm->vmcb->save.gdtr.limit;
  620. dt->base = svm->vmcb->save.gdtr.base;
  621. }
  622. static void svm_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  623. {
  624. struct vcpu_svm *svm = to_svm(vcpu);
  625. svm->vmcb->save.gdtr.limit = dt->limit;
  626. svm->vmcb->save.gdtr.base = dt->base ;
  627. }
  628. static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
  629. {
  630. }
  631. static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  632. {
  633. struct vcpu_svm *svm = to_svm(vcpu);
  634. #ifdef CONFIG_X86_64
  635. if (vcpu->shadow_efer & KVM_EFER_LME) {
  636. if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
  637. vcpu->shadow_efer |= KVM_EFER_LMA;
  638. svm->vmcb->save.efer |= KVM_EFER_LMA | KVM_EFER_LME;
  639. }
  640. if (is_paging(vcpu) && !(cr0 & X86_CR0_PG) ) {
  641. vcpu->shadow_efer &= ~KVM_EFER_LMA;
  642. svm->vmcb->save.efer &= ~(KVM_EFER_LMA | KVM_EFER_LME);
  643. }
  644. }
  645. #endif
  646. if ((vcpu->cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) {
  647. svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR);
  648. vcpu->fpu_active = 1;
  649. }
  650. vcpu->cr0 = cr0;
  651. cr0 |= X86_CR0_PG | X86_CR0_WP;
  652. cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
  653. svm->vmcb->save.cr0 = cr0;
  654. }
  655. static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  656. {
  657. vcpu->cr4 = cr4;
  658. to_svm(vcpu)->vmcb->save.cr4 = cr4 | X86_CR4_PAE;
  659. }
  660. static void svm_set_segment(struct kvm_vcpu *vcpu,
  661. struct kvm_segment *var, int seg)
  662. {
  663. struct vcpu_svm *svm = to_svm(vcpu);
  664. struct vmcb_seg *s = svm_seg(vcpu, seg);
  665. s->base = var->base;
  666. s->limit = var->limit;
  667. s->selector = var->selector;
  668. if (var->unusable)
  669. s->attrib = 0;
  670. else {
  671. s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
  672. s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
  673. s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
  674. s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
  675. s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
  676. s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
  677. s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
  678. s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
  679. }
  680. if (seg == VCPU_SREG_CS)
  681. svm->vmcb->save.cpl
  682. = (svm->vmcb->save.cs.attrib
  683. >> SVM_SELECTOR_DPL_SHIFT) & 3;
  684. }
  685. /* FIXME:
  686. svm(vcpu)->vmcb->control.int_ctl &= ~V_TPR_MASK;
  687. svm(vcpu)->vmcb->control.int_ctl |= (sregs->cr8 & V_TPR_MASK);
  688. */
  689. static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg)
  690. {
  691. return -EOPNOTSUPP;
  692. }
  693. static int svm_get_irq(struct kvm_vcpu *vcpu)
  694. {
  695. struct vcpu_svm *svm = to_svm(vcpu);
  696. u32 exit_int_info = svm->vmcb->control.exit_int_info;
  697. if (is_external_interrupt(exit_int_info))
  698. return exit_int_info & SVM_EVTINJ_VEC_MASK;
  699. return -1;
  700. }
  701. static void load_host_msrs(struct kvm_vcpu *vcpu)
  702. {
  703. #ifdef CONFIG_X86_64
  704. wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
  705. #endif
  706. }
  707. static void save_host_msrs(struct kvm_vcpu *vcpu)
  708. {
  709. #ifdef CONFIG_X86_64
  710. rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
  711. #endif
  712. }
  713. static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *svm_data)
  714. {
  715. if (svm_data->next_asid > svm_data->max_asid) {
  716. ++svm_data->asid_generation;
  717. svm_data->next_asid = 1;
  718. svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
  719. }
  720. svm->vcpu.cpu = svm_data->cpu;
  721. svm->asid_generation = svm_data->asid_generation;
  722. svm->vmcb->control.asid = svm_data->next_asid++;
  723. }
  724. static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr)
  725. {
  726. return to_svm(vcpu)->db_regs[dr];
  727. }
  728. static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value,
  729. int *exception)
  730. {
  731. struct vcpu_svm *svm = to_svm(vcpu);
  732. *exception = 0;
  733. if (svm->vmcb->save.dr7 & DR7_GD_MASK) {
  734. svm->vmcb->save.dr7 &= ~DR7_GD_MASK;
  735. svm->vmcb->save.dr6 |= DR6_BD_MASK;
  736. *exception = DB_VECTOR;
  737. return;
  738. }
  739. switch (dr) {
  740. case 0 ... 3:
  741. svm->db_regs[dr] = value;
  742. return;
  743. case 4 ... 5:
  744. if (vcpu->cr4 & X86_CR4_DE) {
  745. *exception = UD_VECTOR;
  746. return;
  747. }
  748. case 7: {
  749. if (value & ~((1ULL << 32) - 1)) {
  750. *exception = GP_VECTOR;
  751. return;
  752. }
  753. svm->vmcb->save.dr7 = value;
  754. return;
  755. }
  756. default:
  757. printk(KERN_DEBUG "%s: unexpected dr %u\n",
  758. __FUNCTION__, dr);
  759. *exception = UD_VECTOR;
  760. return;
  761. }
  762. }
  763. static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  764. {
  765. u32 exit_int_info = svm->vmcb->control.exit_int_info;
  766. struct kvm *kvm = svm->vcpu.kvm;
  767. u64 fault_address;
  768. u32 error_code;
  769. enum emulation_result er;
  770. int r;
  771. if (!irqchip_in_kernel(kvm) &&
  772. is_external_interrupt(exit_int_info))
  773. push_irq(&svm->vcpu, exit_int_info & SVM_EVTINJ_VEC_MASK);
  774. mutex_lock(&kvm->lock);
  775. fault_address = svm->vmcb->control.exit_info_2;
  776. error_code = svm->vmcb->control.exit_info_1;
  777. r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
  778. if (r < 0) {
  779. mutex_unlock(&kvm->lock);
  780. return r;
  781. }
  782. if (!r) {
  783. mutex_unlock(&kvm->lock);
  784. return 1;
  785. }
  786. er = emulate_instruction(&svm->vcpu, kvm_run, fault_address,
  787. error_code);
  788. mutex_unlock(&kvm->lock);
  789. switch (er) {
  790. case EMULATE_DONE:
  791. return 1;
  792. case EMULATE_DO_MMIO:
  793. ++svm->vcpu.stat.mmio_exits;
  794. return 0;
  795. case EMULATE_FAIL:
  796. kvm_report_emulation_failure(&svm->vcpu, "pagetable");
  797. break;
  798. default:
  799. BUG();
  800. }
  801. kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
  802. return 0;
  803. }
  804. static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  805. {
  806. svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR);
  807. if (!(svm->vcpu.cr0 & X86_CR0_TS))
  808. svm->vmcb->save.cr0 &= ~X86_CR0_TS;
  809. svm->vcpu.fpu_active = 1;
  810. return 1;
  811. }
  812. static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  813. {
  814. /*
  815. * VMCB is undefined after a SHUTDOWN intercept
  816. * so reinitialize it.
  817. */
  818. clear_page(svm->vmcb);
  819. init_vmcb(svm->vmcb);
  820. kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
  821. return 0;
  822. }
  823. static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  824. {
  825. u32 io_info = svm->vmcb->control.exit_info_1; //address size bug?
  826. int size, down, in, string, rep;
  827. unsigned port;
  828. ++svm->vcpu.stat.io_exits;
  829. svm->next_rip = svm->vmcb->control.exit_info_2;
  830. string = (io_info & SVM_IOIO_STR_MASK) != 0;
  831. if (string) {
  832. if (emulate_instruction(&svm->vcpu, kvm_run, 0, 0) == EMULATE_DO_MMIO)
  833. return 0;
  834. return 1;
  835. }
  836. in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
  837. port = io_info >> 16;
  838. size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
  839. rep = (io_info & SVM_IOIO_REP_MASK) != 0;
  840. down = (svm->vmcb->save.rflags & X86_EFLAGS_DF) != 0;
  841. return kvm_emulate_pio(&svm->vcpu, kvm_run, in, size, port);
  842. }
  843. static int nop_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  844. {
  845. return 1;
  846. }
  847. static int halt_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  848. {
  849. svm->next_rip = svm->vmcb->save.rip + 1;
  850. skip_emulated_instruction(&svm->vcpu);
  851. return kvm_emulate_halt(&svm->vcpu);
  852. }
  853. static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  854. {
  855. svm->next_rip = svm->vmcb->save.rip + 3;
  856. skip_emulated_instruction(&svm->vcpu);
  857. return kvm_hypercall(&svm->vcpu, kvm_run);
  858. }
  859. static int invalid_op_interception(struct vcpu_svm *svm,
  860. struct kvm_run *kvm_run)
  861. {
  862. inject_ud(&svm->vcpu);
  863. return 1;
  864. }
  865. static int task_switch_interception(struct vcpu_svm *svm,
  866. struct kvm_run *kvm_run)
  867. {
  868. pr_unimpl(&svm->vcpu, "%s: task switch is unsupported\n", __FUNCTION__);
  869. kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
  870. return 0;
  871. }
  872. static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  873. {
  874. svm->next_rip = svm->vmcb->save.rip + 2;
  875. kvm_emulate_cpuid(&svm->vcpu);
  876. return 1;
  877. }
  878. static int emulate_on_interception(struct vcpu_svm *svm,
  879. struct kvm_run *kvm_run)
  880. {
  881. if (emulate_instruction(&svm->vcpu, NULL, 0, 0) != EMULATE_DONE)
  882. pr_unimpl(&svm->vcpu, "%s: failed\n", __FUNCTION__);
  883. return 1;
  884. }
  885. static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
  886. {
  887. struct vcpu_svm *svm = to_svm(vcpu);
  888. switch (ecx) {
  889. case MSR_IA32_TIME_STAMP_COUNTER: {
  890. u64 tsc;
  891. rdtscll(tsc);
  892. *data = svm->vmcb->control.tsc_offset + tsc;
  893. break;
  894. }
  895. case MSR_K6_STAR:
  896. *data = svm->vmcb->save.star;
  897. break;
  898. #ifdef CONFIG_X86_64
  899. case MSR_LSTAR:
  900. *data = svm->vmcb->save.lstar;
  901. break;
  902. case MSR_CSTAR:
  903. *data = svm->vmcb->save.cstar;
  904. break;
  905. case MSR_KERNEL_GS_BASE:
  906. *data = svm->vmcb->save.kernel_gs_base;
  907. break;
  908. case MSR_SYSCALL_MASK:
  909. *data = svm->vmcb->save.sfmask;
  910. break;
  911. #endif
  912. case MSR_IA32_SYSENTER_CS:
  913. *data = svm->vmcb->save.sysenter_cs;
  914. break;
  915. case MSR_IA32_SYSENTER_EIP:
  916. *data = svm->vmcb->save.sysenter_eip;
  917. break;
  918. case MSR_IA32_SYSENTER_ESP:
  919. *data = svm->vmcb->save.sysenter_esp;
  920. break;
  921. default:
  922. return kvm_get_msr_common(vcpu, ecx, data);
  923. }
  924. return 0;
  925. }
  926. static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  927. {
  928. u32 ecx = svm->vcpu.regs[VCPU_REGS_RCX];
  929. u64 data;
  930. if (svm_get_msr(&svm->vcpu, ecx, &data))
  931. svm_inject_gp(&svm->vcpu, 0);
  932. else {
  933. svm->vmcb->save.rax = data & 0xffffffff;
  934. svm->vcpu.regs[VCPU_REGS_RDX] = data >> 32;
  935. svm->next_rip = svm->vmcb->save.rip + 2;
  936. skip_emulated_instruction(&svm->vcpu);
  937. }
  938. return 1;
  939. }
  940. static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
  941. {
  942. struct vcpu_svm *svm = to_svm(vcpu);
  943. switch (ecx) {
  944. case MSR_IA32_TIME_STAMP_COUNTER: {
  945. u64 tsc;
  946. rdtscll(tsc);
  947. svm->vmcb->control.tsc_offset = data - tsc;
  948. break;
  949. }
  950. case MSR_K6_STAR:
  951. svm->vmcb->save.star = data;
  952. break;
  953. #ifdef CONFIG_X86_64
  954. case MSR_LSTAR:
  955. svm->vmcb->save.lstar = data;
  956. break;
  957. case MSR_CSTAR:
  958. svm->vmcb->save.cstar = data;
  959. break;
  960. case MSR_KERNEL_GS_BASE:
  961. svm->vmcb->save.kernel_gs_base = data;
  962. break;
  963. case MSR_SYSCALL_MASK:
  964. svm->vmcb->save.sfmask = data;
  965. break;
  966. #endif
  967. case MSR_IA32_SYSENTER_CS:
  968. svm->vmcb->save.sysenter_cs = data;
  969. break;
  970. case MSR_IA32_SYSENTER_EIP:
  971. svm->vmcb->save.sysenter_eip = data;
  972. break;
  973. case MSR_IA32_SYSENTER_ESP:
  974. svm->vmcb->save.sysenter_esp = data;
  975. break;
  976. default:
  977. return kvm_set_msr_common(vcpu, ecx, data);
  978. }
  979. return 0;
  980. }
  981. static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  982. {
  983. u32 ecx = svm->vcpu.regs[VCPU_REGS_RCX];
  984. u64 data = (svm->vmcb->save.rax & -1u)
  985. | ((u64)(svm->vcpu.regs[VCPU_REGS_RDX] & -1u) << 32);
  986. svm->next_rip = svm->vmcb->save.rip + 2;
  987. if (svm_set_msr(&svm->vcpu, ecx, data))
  988. svm_inject_gp(&svm->vcpu, 0);
  989. else
  990. skip_emulated_instruction(&svm->vcpu);
  991. return 1;
  992. }
  993. static int msr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
  994. {
  995. if (svm->vmcb->control.exit_info_1)
  996. return wrmsr_interception(svm, kvm_run);
  997. else
  998. return rdmsr_interception(svm, kvm_run);
  999. }
  1000. static int interrupt_window_interception(struct vcpu_svm *svm,
  1001. struct kvm_run *kvm_run)
  1002. {
  1003. svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
  1004. svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
  1005. /*
  1006. * If the user space waits to inject interrupts, exit as soon as
  1007. * possible
  1008. */
  1009. if (kvm_run->request_interrupt_window &&
  1010. !svm->vcpu.irq_summary) {
  1011. ++svm->vcpu.stat.irq_window_exits;
  1012. kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
  1013. return 0;
  1014. }
  1015. return 1;
  1016. }
  1017. static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
  1018. struct kvm_run *kvm_run) = {
  1019. [SVM_EXIT_READ_CR0] = emulate_on_interception,
  1020. [SVM_EXIT_READ_CR3] = emulate_on_interception,
  1021. [SVM_EXIT_READ_CR4] = emulate_on_interception,
  1022. /* for now: */
  1023. [SVM_EXIT_WRITE_CR0] = emulate_on_interception,
  1024. [SVM_EXIT_WRITE_CR3] = emulate_on_interception,
  1025. [SVM_EXIT_WRITE_CR4] = emulate_on_interception,
  1026. [SVM_EXIT_READ_DR0] = emulate_on_interception,
  1027. [SVM_EXIT_READ_DR1] = emulate_on_interception,
  1028. [SVM_EXIT_READ_DR2] = emulate_on_interception,
  1029. [SVM_EXIT_READ_DR3] = emulate_on_interception,
  1030. [SVM_EXIT_WRITE_DR0] = emulate_on_interception,
  1031. [SVM_EXIT_WRITE_DR1] = emulate_on_interception,
  1032. [SVM_EXIT_WRITE_DR2] = emulate_on_interception,
  1033. [SVM_EXIT_WRITE_DR3] = emulate_on_interception,
  1034. [SVM_EXIT_WRITE_DR5] = emulate_on_interception,
  1035. [SVM_EXIT_WRITE_DR7] = emulate_on_interception,
  1036. [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception,
  1037. [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception,
  1038. [SVM_EXIT_INTR] = nop_on_interception,
  1039. [SVM_EXIT_NMI] = nop_on_interception,
  1040. [SVM_EXIT_SMI] = nop_on_interception,
  1041. [SVM_EXIT_INIT] = nop_on_interception,
  1042. [SVM_EXIT_VINTR] = interrupt_window_interception,
  1043. /* [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception, */
  1044. [SVM_EXIT_CPUID] = cpuid_interception,
  1045. [SVM_EXIT_INVD] = emulate_on_interception,
  1046. [SVM_EXIT_HLT] = halt_interception,
  1047. [SVM_EXIT_INVLPG] = emulate_on_interception,
  1048. [SVM_EXIT_INVLPGA] = invalid_op_interception,
  1049. [SVM_EXIT_IOIO] = io_interception,
  1050. [SVM_EXIT_MSR] = msr_interception,
  1051. [SVM_EXIT_TASK_SWITCH] = task_switch_interception,
  1052. [SVM_EXIT_SHUTDOWN] = shutdown_interception,
  1053. [SVM_EXIT_VMRUN] = invalid_op_interception,
  1054. [SVM_EXIT_VMMCALL] = vmmcall_interception,
  1055. [SVM_EXIT_VMLOAD] = invalid_op_interception,
  1056. [SVM_EXIT_VMSAVE] = invalid_op_interception,
  1057. [SVM_EXIT_STGI] = invalid_op_interception,
  1058. [SVM_EXIT_CLGI] = invalid_op_interception,
  1059. [SVM_EXIT_SKINIT] = invalid_op_interception,
  1060. [SVM_EXIT_WBINVD] = emulate_on_interception,
  1061. [SVM_EXIT_MONITOR] = invalid_op_interception,
  1062. [SVM_EXIT_MWAIT] = invalid_op_interception,
  1063. };
  1064. static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
  1065. {
  1066. struct vcpu_svm *svm = to_svm(vcpu);
  1067. u32 exit_code = svm->vmcb->control.exit_code;
  1068. kvm_reput_irq(svm);
  1069. if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
  1070. kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
  1071. kvm_run->fail_entry.hardware_entry_failure_reason
  1072. = svm->vmcb->control.exit_code;
  1073. return 0;
  1074. }
  1075. if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
  1076. exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR)
  1077. printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
  1078. "exit_code 0x%x\n",
  1079. __FUNCTION__, svm->vmcb->control.exit_int_info,
  1080. exit_code);
  1081. if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
  1082. || svm_exit_handlers[exit_code] == 0) {
  1083. kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
  1084. kvm_run->hw.hardware_exit_reason = exit_code;
  1085. return 0;
  1086. }
  1087. return svm_exit_handlers[exit_code](svm, kvm_run);
  1088. }
  1089. static void reload_tss(struct kvm_vcpu *vcpu)
  1090. {
  1091. int cpu = raw_smp_processor_id();
  1092. struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
  1093. svm_data->tss_desc->type = 9; //available 32/64-bit TSS
  1094. load_TR_desc();
  1095. }
  1096. static void pre_svm_run(struct vcpu_svm *svm)
  1097. {
  1098. int cpu = raw_smp_processor_id();
  1099. struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
  1100. svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
  1101. if (svm->vcpu.cpu != cpu ||
  1102. svm->asid_generation != svm_data->asid_generation)
  1103. new_asid(svm, svm_data);
  1104. }
  1105. static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
  1106. {
  1107. struct vmcb_control_area *control;
  1108. control = &svm->vmcb->control;
  1109. control->int_vector = irq;
  1110. control->int_ctl &= ~V_INTR_PRIO_MASK;
  1111. control->int_ctl |= V_IRQ_MASK |
  1112. ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
  1113. }
  1114. static void svm_set_irq(struct kvm_vcpu *vcpu, int irq)
  1115. {
  1116. struct vcpu_svm *svm = to_svm(vcpu);
  1117. svm_inject_irq(svm, irq);
  1118. }
  1119. static void svm_intr_assist(struct kvm_vcpu *vcpu)
  1120. {
  1121. struct vcpu_svm *svm = to_svm(vcpu);
  1122. struct vmcb *vmcb = svm->vmcb;
  1123. int intr_vector = -1;
  1124. kvm_inject_pending_timer_irqs(vcpu);
  1125. if ((vmcb->control.exit_int_info & SVM_EVTINJ_VALID) &&
  1126. ((vmcb->control.exit_int_info & SVM_EVTINJ_TYPE_MASK) == 0)) {
  1127. intr_vector = vmcb->control.exit_int_info &
  1128. SVM_EVTINJ_VEC_MASK;
  1129. vmcb->control.exit_int_info = 0;
  1130. svm_inject_irq(svm, intr_vector);
  1131. return;
  1132. }
  1133. if (vmcb->control.int_ctl & V_IRQ_MASK)
  1134. return;
  1135. if (!kvm_cpu_has_interrupt(vcpu))
  1136. return;
  1137. if (!(vmcb->save.rflags & X86_EFLAGS_IF) ||
  1138. (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) ||
  1139. (vmcb->control.event_inj & SVM_EVTINJ_VALID)) {
  1140. /* unable to deliver irq, set pending irq */
  1141. vmcb->control.intercept |= (1ULL << INTERCEPT_VINTR);
  1142. svm_inject_irq(svm, 0x0);
  1143. return;
  1144. }
  1145. /* Okay, we can deliver the interrupt: grab it and update PIC state. */
  1146. intr_vector = kvm_cpu_get_interrupt(vcpu);
  1147. svm_inject_irq(svm, intr_vector);
  1148. kvm_timer_intr_post(vcpu, intr_vector);
  1149. }
  1150. static void kvm_reput_irq(struct vcpu_svm *svm)
  1151. {
  1152. struct vmcb_control_area *control = &svm->vmcb->control;
  1153. if ((control->int_ctl & V_IRQ_MASK)
  1154. && !irqchip_in_kernel(svm->vcpu.kvm)) {
  1155. control->int_ctl &= ~V_IRQ_MASK;
  1156. push_irq(&svm->vcpu, control->int_vector);
  1157. }
  1158. svm->vcpu.interrupt_window_open =
  1159. !(control->int_state & SVM_INTERRUPT_SHADOW_MASK);
  1160. }
  1161. static void svm_do_inject_vector(struct vcpu_svm *svm)
  1162. {
  1163. struct kvm_vcpu *vcpu = &svm->vcpu;
  1164. int word_index = __ffs(vcpu->irq_summary);
  1165. int bit_index = __ffs(vcpu->irq_pending[word_index]);
  1166. int irq = word_index * BITS_PER_LONG + bit_index;
  1167. clear_bit(bit_index, &vcpu->irq_pending[word_index]);
  1168. if (!vcpu->irq_pending[word_index])
  1169. clear_bit(word_index, &vcpu->irq_summary);
  1170. svm_inject_irq(svm, irq);
  1171. }
  1172. static void do_interrupt_requests(struct kvm_vcpu *vcpu,
  1173. struct kvm_run *kvm_run)
  1174. {
  1175. struct vcpu_svm *svm = to_svm(vcpu);
  1176. struct vmcb_control_area *control = &svm->vmcb->control;
  1177. svm->vcpu.interrupt_window_open =
  1178. (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) &&
  1179. (svm->vmcb->save.rflags & X86_EFLAGS_IF));
  1180. if (svm->vcpu.interrupt_window_open && svm->vcpu.irq_summary)
  1181. /*
  1182. * If interrupts enabled, and not blocked by sti or mov ss. Good.
  1183. */
  1184. svm_do_inject_vector(svm);
  1185. /*
  1186. * Interrupts blocked. Wait for unblock.
  1187. */
  1188. if (!svm->vcpu.interrupt_window_open &&
  1189. (svm->vcpu.irq_summary || kvm_run->request_interrupt_window)) {
  1190. control->intercept |= 1ULL << INTERCEPT_VINTR;
  1191. } else
  1192. control->intercept &= ~(1ULL << INTERCEPT_VINTR);
  1193. }
  1194. static void save_db_regs(unsigned long *db_regs)
  1195. {
  1196. asm volatile ("mov %%dr0, %0" : "=r"(db_regs[0]));
  1197. asm volatile ("mov %%dr1, %0" : "=r"(db_regs[1]));
  1198. asm volatile ("mov %%dr2, %0" : "=r"(db_regs[2]));
  1199. asm volatile ("mov %%dr3, %0" : "=r"(db_regs[3]));
  1200. }
  1201. static void load_db_regs(unsigned long *db_regs)
  1202. {
  1203. asm volatile ("mov %0, %%dr0" : : "r"(db_regs[0]));
  1204. asm volatile ("mov %0, %%dr1" : : "r"(db_regs[1]));
  1205. asm volatile ("mov %0, %%dr2" : : "r"(db_regs[2]));
  1206. asm volatile ("mov %0, %%dr3" : : "r"(db_regs[3]));
  1207. }
  1208. static void svm_flush_tlb(struct kvm_vcpu *vcpu)
  1209. {
  1210. force_new_asid(vcpu);
  1211. }
  1212. static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
  1213. {
  1214. }
  1215. static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1216. {
  1217. struct vcpu_svm *svm = to_svm(vcpu);
  1218. u16 fs_selector;
  1219. u16 gs_selector;
  1220. u16 ldt_selector;
  1221. pre_svm_run(svm);
  1222. save_host_msrs(vcpu);
  1223. fs_selector = read_fs();
  1224. gs_selector = read_gs();
  1225. ldt_selector = read_ldt();
  1226. svm->host_cr2 = kvm_read_cr2();
  1227. svm->host_dr6 = read_dr6();
  1228. svm->host_dr7 = read_dr7();
  1229. svm->vmcb->save.cr2 = vcpu->cr2;
  1230. if (svm->vmcb->save.dr7 & 0xff) {
  1231. write_dr7(0);
  1232. save_db_regs(svm->host_db_regs);
  1233. load_db_regs(svm->db_regs);
  1234. }
  1235. clgi();
  1236. local_irq_enable();
  1237. asm volatile (
  1238. #ifdef CONFIG_X86_64
  1239. "push %%rbx; push %%rcx; push %%rdx;"
  1240. "push %%rsi; push %%rdi; push %%rbp;"
  1241. "push %%r8; push %%r9; push %%r10; push %%r11;"
  1242. "push %%r12; push %%r13; push %%r14; push %%r15;"
  1243. #else
  1244. "push %%ebx; push %%ecx; push %%edx;"
  1245. "push %%esi; push %%edi; push %%ebp;"
  1246. #endif
  1247. #ifdef CONFIG_X86_64
  1248. "mov %c[rbx](%[svm]), %%rbx \n\t"
  1249. "mov %c[rcx](%[svm]), %%rcx \n\t"
  1250. "mov %c[rdx](%[svm]), %%rdx \n\t"
  1251. "mov %c[rsi](%[svm]), %%rsi \n\t"
  1252. "mov %c[rdi](%[svm]), %%rdi \n\t"
  1253. "mov %c[rbp](%[svm]), %%rbp \n\t"
  1254. "mov %c[r8](%[svm]), %%r8 \n\t"
  1255. "mov %c[r9](%[svm]), %%r9 \n\t"
  1256. "mov %c[r10](%[svm]), %%r10 \n\t"
  1257. "mov %c[r11](%[svm]), %%r11 \n\t"
  1258. "mov %c[r12](%[svm]), %%r12 \n\t"
  1259. "mov %c[r13](%[svm]), %%r13 \n\t"
  1260. "mov %c[r14](%[svm]), %%r14 \n\t"
  1261. "mov %c[r15](%[svm]), %%r15 \n\t"
  1262. #else
  1263. "mov %c[rbx](%[svm]), %%ebx \n\t"
  1264. "mov %c[rcx](%[svm]), %%ecx \n\t"
  1265. "mov %c[rdx](%[svm]), %%edx \n\t"
  1266. "mov %c[rsi](%[svm]), %%esi \n\t"
  1267. "mov %c[rdi](%[svm]), %%edi \n\t"
  1268. "mov %c[rbp](%[svm]), %%ebp \n\t"
  1269. #endif
  1270. #ifdef CONFIG_X86_64
  1271. /* Enter guest mode */
  1272. "push %%rax \n\t"
  1273. "mov %c[vmcb](%[svm]), %%rax \n\t"
  1274. SVM_VMLOAD "\n\t"
  1275. SVM_VMRUN "\n\t"
  1276. SVM_VMSAVE "\n\t"
  1277. "pop %%rax \n\t"
  1278. #else
  1279. /* Enter guest mode */
  1280. "push %%eax \n\t"
  1281. "mov %c[vmcb](%[svm]), %%eax \n\t"
  1282. SVM_VMLOAD "\n\t"
  1283. SVM_VMRUN "\n\t"
  1284. SVM_VMSAVE "\n\t"
  1285. "pop %%eax \n\t"
  1286. #endif
  1287. /* Save guest registers, load host registers */
  1288. #ifdef CONFIG_X86_64
  1289. "mov %%rbx, %c[rbx](%[svm]) \n\t"
  1290. "mov %%rcx, %c[rcx](%[svm]) \n\t"
  1291. "mov %%rdx, %c[rdx](%[svm]) \n\t"
  1292. "mov %%rsi, %c[rsi](%[svm]) \n\t"
  1293. "mov %%rdi, %c[rdi](%[svm]) \n\t"
  1294. "mov %%rbp, %c[rbp](%[svm]) \n\t"
  1295. "mov %%r8, %c[r8](%[svm]) \n\t"
  1296. "mov %%r9, %c[r9](%[svm]) \n\t"
  1297. "mov %%r10, %c[r10](%[svm]) \n\t"
  1298. "mov %%r11, %c[r11](%[svm]) \n\t"
  1299. "mov %%r12, %c[r12](%[svm]) \n\t"
  1300. "mov %%r13, %c[r13](%[svm]) \n\t"
  1301. "mov %%r14, %c[r14](%[svm]) \n\t"
  1302. "mov %%r15, %c[r15](%[svm]) \n\t"
  1303. "pop %%r15; pop %%r14; pop %%r13; pop %%r12;"
  1304. "pop %%r11; pop %%r10; pop %%r9; pop %%r8;"
  1305. "pop %%rbp; pop %%rdi; pop %%rsi;"
  1306. "pop %%rdx; pop %%rcx; pop %%rbx; \n\t"
  1307. #else
  1308. "mov %%ebx, %c[rbx](%[svm]) \n\t"
  1309. "mov %%ecx, %c[rcx](%[svm]) \n\t"
  1310. "mov %%edx, %c[rdx](%[svm]) \n\t"
  1311. "mov %%esi, %c[rsi](%[svm]) \n\t"
  1312. "mov %%edi, %c[rdi](%[svm]) \n\t"
  1313. "mov %%ebp, %c[rbp](%[svm]) \n\t"
  1314. "pop %%ebp; pop %%edi; pop %%esi;"
  1315. "pop %%edx; pop %%ecx; pop %%ebx; \n\t"
  1316. #endif
  1317. :
  1318. : [svm]"a"(svm),
  1319. [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
  1320. [rbx]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RBX])),
  1321. [rcx]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RCX])),
  1322. [rdx]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RDX])),
  1323. [rsi]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RSI])),
  1324. [rdi]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RDI])),
  1325. [rbp]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RBP]))
  1326. #ifdef CONFIG_X86_64
  1327. ,[r8 ]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R8])),
  1328. [r9 ]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R9 ])),
  1329. [r10]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R10])),
  1330. [r11]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R11])),
  1331. [r12]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R12])),
  1332. [r13]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R13])),
  1333. [r14]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R14])),
  1334. [r15]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R15]))
  1335. #endif
  1336. : "cc", "memory" );
  1337. if ((svm->vmcb->save.dr7 & 0xff))
  1338. load_db_regs(svm->host_db_regs);
  1339. vcpu->cr2 = svm->vmcb->save.cr2;
  1340. write_dr6(svm->host_dr6);
  1341. write_dr7(svm->host_dr7);
  1342. kvm_write_cr2(svm->host_cr2);
  1343. load_fs(fs_selector);
  1344. load_gs(gs_selector);
  1345. load_ldt(ldt_selector);
  1346. load_host_msrs(vcpu);
  1347. reload_tss(vcpu);
  1348. local_irq_disable();
  1349. stgi();
  1350. svm->next_rip = 0;
  1351. }
  1352. static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
  1353. {
  1354. struct vcpu_svm *svm = to_svm(vcpu);
  1355. svm->vmcb->save.cr3 = root;
  1356. force_new_asid(vcpu);
  1357. if (vcpu->fpu_active) {
  1358. svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR);
  1359. svm->vmcb->save.cr0 |= X86_CR0_TS;
  1360. vcpu->fpu_active = 0;
  1361. }
  1362. }
  1363. static void svm_inject_page_fault(struct kvm_vcpu *vcpu,
  1364. unsigned long addr,
  1365. uint32_t err_code)
  1366. {
  1367. struct vcpu_svm *svm = to_svm(vcpu);
  1368. uint32_t exit_int_info = svm->vmcb->control.exit_int_info;
  1369. ++vcpu->stat.pf_guest;
  1370. if (is_page_fault(exit_int_info)) {
  1371. svm->vmcb->control.event_inj_err = 0;
  1372. svm->vmcb->control.event_inj = SVM_EVTINJ_VALID |
  1373. SVM_EVTINJ_VALID_ERR |
  1374. SVM_EVTINJ_TYPE_EXEPT |
  1375. DF_VECTOR;
  1376. return;
  1377. }
  1378. vcpu->cr2 = addr;
  1379. svm->vmcb->save.cr2 = addr;
  1380. svm->vmcb->control.event_inj = SVM_EVTINJ_VALID |
  1381. SVM_EVTINJ_VALID_ERR |
  1382. SVM_EVTINJ_TYPE_EXEPT |
  1383. PF_VECTOR;
  1384. svm->vmcb->control.event_inj_err = err_code;
  1385. }
  1386. static int is_disabled(void)
  1387. {
  1388. u64 vm_cr;
  1389. rdmsrl(MSR_VM_CR, vm_cr);
  1390. if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
  1391. return 1;
  1392. return 0;
  1393. }
  1394. static void
  1395. svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
  1396. {
  1397. /*
  1398. * Patch in the VMMCALL instruction:
  1399. */
  1400. hypercall[0] = 0x0f;
  1401. hypercall[1] = 0x01;
  1402. hypercall[2] = 0xd9;
  1403. hypercall[3] = 0xc3;
  1404. }
  1405. static void svm_check_processor_compat(void *rtn)
  1406. {
  1407. *(int *)rtn = 0;
  1408. }
  1409. static struct kvm_x86_ops svm_x86_ops = {
  1410. .cpu_has_kvm_support = has_svm,
  1411. .disabled_by_bios = is_disabled,
  1412. .hardware_setup = svm_hardware_setup,
  1413. .hardware_unsetup = svm_hardware_unsetup,
  1414. .check_processor_compatibility = svm_check_processor_compat,
  1415. .hardware_enable = svm_hardware_enable,
  1416. .hardware_disable = svm_hardware_disable,
  1417. .vcpu_create = svm_create_vcpu,
  1418. .vcpu_free = svm_free_vcpu,
  1419. .vcpu_reset = svm_vcpu_reset,
  1420. .prepare_guest_switch = svm_prepare_guest_switch,
  1421. .vcpu_load = svm_vcpu_load,
  1422. .vcpu_put = svm_vcpu_put,
  1423. .vcpu_decache = svm_vcpu_decache,
  1424. .set_guest_debug = svm_guest_debug,
  1425. .get_msr = svm_get_msr,
  1426. .set_msr = svm_set_msr,
  1427. .get_segment_base = svm_get_segment_base,
  1428. .get_segment = svm_get_segment,
  1429. .set_segment = svm_set_segment,
  1430. .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
  1431. .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
  1432. .set_cr0 = svm_set_cr0,
  1433. .set_cr3 = svm_set_cr3,
  1434. .set_cr4 = svm_set_cr4,
  1435. .set_efer = svm_set_efer,
  1436. .get_idt = svm_get_idt,
  1437. .set_idt = svm_set_idt,
  1438. .get_gdt = svm_get_gdt,
  1439. .set_gdt = svm_set_gdt,
  1440. .get_dr = svm_get_dr,
  1441. .set_dr = svm_set_dr,
  1442. .cache_regs = svm_cache_regs,
  1443. .decache_regs = svm_decache_regs,
  1444. .get_rflags = svm_get_rflags,
  1445. .set_rflags = svm_set_rflags,
  1446. .tlb_flush = svm_flush_tlb,
  1447. .inject_page_fault = svm_inject_page_fault,
  1448. .inject_gp = svm_inject_gp,
  1449. .run = svm_vcpu_run,
  1450. .handle_exit = handle_exit,
  1451. .skip_emulated_instruction = skip_emulated_instruction,
  1452. .patch_hypercall = svm_patch_hypercall,
  1453. .get_irq = svm_get_irq,
  1454. .set_irq = svm_set_irq,
  1455. .inject_pending_irq = svm_intr_assist,
  1456. .inject_pending_vectors = do_interrupt_requests,
  1457. };
  1458. static int __init svm_init(void)
  1459. {
  1460. return kvm_init_x86(&svm_x86_ops, sizeof(struct vcpu_svm),
  1461. THIS_MODULE);
  1462. }
  1463. static void __exit svm_exit(void)
  1464. {
  1465. kvm_exit_x86();
  1466. }
  1467. module_init(svm_init)
  1468. module_exit(svm_exit)