kvm_main.c 34 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. *
  9. * Authors:
  10. * Avi Kivity <avi@qumranet.com>
  11. * Yaniv Kamay <yaniv@qumranet.com>
  12. *
  13. * This work is licensed under the terms of the GNU GPL, version 2. See
  14. * the COPYING file in the top-level directory.
  15. *
  16. */
  17. #include "kvm.h"
  18. #include "x86.h"
  19. #include "x86_emulate.h"
  20. #include "irq.h"
  21. #include <linux/kvm.h>
  22. #include <linux/module.h>
  23. #include <linux/errno.h>
  24. #include <linux/percpu.h>
  25. #include <linux/gfp.h>
  26. #include <linux/mm.h>
  27. #include <linux/miscdevice.h>
  28. #include <linux/vmalloc.h>
  29. #include <linux/reboot.h>
  30. #include <linux/debugfs.h>
  31. #include <linux/highmem.h>
  32. #include <linux/file.h>
  33. #include <linux/sysdev.h>
  34. #include <linux/cpu.h>
  35. #include <linux/sched.h>
  36. #include <linux/cpumask.h>
  37. #include <linux/smp.h>
  38. #include <linux/anon_inodes.h>
  39. #include <linux/profile.h>
  40. #include <linux/kvm_para.h>
  41. #include <linux/pagemap.h>
  42. #include <linux/mman.h>
  43. #include <asm/processor.h>
  44. #include <asm/msr.h>
  45. #include <asm/io.h>
  46. #include <asm/uaccess.h>
  47. #include <asm/desc.h>
  48. MODULE_AUTHOR("Qumranet");
  49. MODULE_LICENSE("GPL");
  50. static DEFINE_SPINLOCK(kvm_lock);
  51. static LIST_HEAD(vm_list);
  52. static cpumask_t cpus_hardware_enabled;
  53. struct kvm_x86_ops *kvm_x86_ops;
  54. struct kmem_cache *kvm_vcpu_cache;
  55. EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  56. static __read_mostly struct preempt_ops kvm_preempt_ops;
  57. static struct dentry *debugfs_dir;
  58. static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  59. unsigned long arg);
  60. static inline int valid_vcpu(int n)
  61. {
  62. return likely(n >= 0 && n < KVM_MAX_VCPUS);
  63. }
  64. /*
  65. * Switches to specified vcpu, until a matching vcpu_put()
  66. */
  67. void vcpu_load(struct kvm_vcpu *vcpu)
  68. {
  69. int cpu;
  70. mutex_lock(&vcpu->mutex);
  71. cpu = get_cpu();
  72. preempt_notifier_register(&vcpu->preempt_notifier);
  73. kvm_arch_vcpu_load(vcpu, cpu);
  74. put_cpu();
  75. }
  76. void vcpu_put(struct kvm_vcpu *vcpu)
  77. {
  78. preempt_disable();
  79. kvm_arch_vcpu_put(vcpu);
  80. preempt_notifier_unregister(&vcpu->preempt_notifier);
  81. preempt_enable();
  82. mutex_unlock(&vcpu->mutex);
  83. }
  84. static void ack_flush(void *_completed)
  85. {
  86. }
  87. void kvm_flush_remote_tlbs(struct kvm *kvm)
  88. {
  89. int i, cpu;
  90. cpumask_t cpus;
  91. struct kvm_vcpu *vcpu;
  92. cpus_clear(cpus);
  93. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  94. vcpu = kvm->vcpus[i];
  95. if (!vcpu)
  96. continue;
  97. if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
  98. continue;
  99. cpu = vcpu->cpu;
  100. if (cpu != -1 && cpu != raw_smp_processor_id())
  101. cpu_set(cpu, cpus);
  102. }
  103. smp_call_function_mask(cpus, ack_flush, NULL, 1);
  104. }
  105. int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
  106. {
  107. struct page *page;
  108. int r;
  109. mutex_init(&vcpu->mutex);
  110. vcpu->cpu = -1;
  111. vcpu->mmu.root_hpa = INVALID_PAGE;
  112. vcpu->kvm = kvm;
  113. vcpu->vcpu_id = id;
  114. if (!irqchip_in_kernel(kvm) || id == 0)
  115. vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
  116. else
  117. vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED;
  118. init_waitqueue_head(&vcpu->wq);
  119. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  120. if (!page) {
  121. r = -ENOMEM;
  122. goto fail;
  123. }
  124. vcpu->run = page_address(page);
  125. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  126. if (!page) {
  127. r = -ENOMEM;
  128. goto fail_free_run;
  129. }
  130. vcpu->pio_data = page_address(page);
  131. r = kvm_mmu_create(vcpu);
  132. if (r < 0)
  133. goto fail_free_pio_data;
  134. if (irqchip_in_kernel(kvm)) {
  135. r = kvm_create_lapic(vcpu);
  136. if (r < 0)
  137. goto fail_mmu_destroy;
  138. }
  139. return 0;
  140. fail_mmu_destroy:
  141. kvm_mmu_destroy(vcpu);
  142. fail_free_pio_data:
  143. free_page((unsigned long)vcpu->pio_data);
  144. fail_free_run:
  145. free_page((unsigned long)vcpu->run);
  146. fail:
  147. return r;
  148. }
  149. EXPORT_SYMBOL_GPL(kvm_vcpu_init);
  150. void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
  151. {
  152. kvm_free_lapic(vcpu);
  153. kvm_mmu_destroy(vcpu);
  154. free_page((unsigned long)vcpu->pio_data);
  155. free_page((unsigned long)vcpu->run);
  156. }
  157. EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
  158. static struct kvm *kvm_create_vm(void)
  159. {
  160. struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
  161. if (!kvm)
  162. return ERR_PTR(-ENOMEM);
  163. kvm_io_bus_init(&kvm->pio_bus);
  164. mutex_init(&kvm->lock);
  165. INIT_LIST_HEAD(&kvm->active_mmu_pages);
  166. kvm_io_bus_init(&kvm->mmio_bus);
  167. spin_lock(&kvm_lock);
  168. list_add(&kvm->vm_list, &vm_list);
  169. spin_unlock(&kvm_lock);
  170. return kvm;
  171. }
  172. /*
  173. * Free any memory in @free but not in @dont.
  174. */
  175. static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
  176. struct kvm_memory_slot *dont)
  177. {
  178. if (!dont || free->rmap != dont->rmap)
  179. vfree(free->rmap);
  180. if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
  181. vfree(free->dirty_bitmap);
  182. free->npages = 0;
  183. free->dirty_bitmap = NULL;
  184. free->rmap = NULL;
  185. }
  186. static void kvm_free_physmem(struct kvm *kvm)
  187. {
  188. int i;
  189. for (i = 0; i < kvm->nmemslots; ++i)
  190. kvm_free_physmem_slot(&kvm->memslots[i], NULL);
  191. }
  192. static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
  193. {
  194. vcpu_load(vcpu);
  195. kvm_mmu_unload(vcpu);
  196. vcpu_put(vcpu);
  197. }
  198. static void kvm_free_vcpus(struct kvm *kvm)
  199. {
  200. unsigned int i;
  201. /*
  202. * Unpin any mmu pages first.
  203. */
  204. for (i = 0; i < KVM_MAX_VCPUS; ++i)
  205. if (kvm->vcpus[i])
  206. kvm_unload_vcpu_mmu(kvm->vcpus[i]);
  207. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  208. if (kvm->vcpus[i]) {
  209. kvm_x86_ops->vcpu_free(kvm->vcpus[i]);
  210. kvm->vcpus[i] = NULL;
  211. }
  212. }
  213. }
  214. static void kvm_destroy_vm(struct kvm *kvm)
  215. {
  216. spin_lock(&kvm_lock);
  217. list_del(&kvm->vm_list);
  218. spin_unlock(&kvm_lock);
  219. kvm_io_bus_destroy(&kvm->pio_bus);
  220. kvm_io_bus_destroy(&kvm->mmio_bus);
  221. kfree(kvm->vpic);
  222. kfree(kvm->vioapic);
  223. kvm_free_vcpus(kvm);
  224. kvm_free_physmem(kvm);
  225. kfree(kvm);
  226. }
  227. static int kvm_vm_release(struct inode *inode, struct file *filp)
  228. {
  229. struct kvm *kvm = filp->private_data;
  230. kvm_destroy_vm(kvm);
  231. return 0;
  232. }
  233. /*
  234. * Allocate some memory and give it an address in the guest physical address
  235. * space.
  236. *
  237. * Discontiguous memory is allowed, mostly for framebuffers.
  238. *
  239. * Must be called holding kvm->lock.
  240. */
  241. int __kvm_set_memory_region(struct kvm *kvm,
  242. struct kvm_userspace_memory_region *mem,
  243. int user_alloc)
  244. {
  245. int r;
  246. gfn_t base_gfn;
  247. unsigned long npages;
  248. unsigned long i;
  249. struct kvm_memory_slot *memslot;
  250. struct kvm_memory_slot old, new;
  251. r = -EINVAL;
  252. /* General sanity checks */
  253. if (mem->memory_size & (PAGE_SIZE - 1))
  254. goto out;
  255. if (mem->guest_phys_addr & (PAGE_SIZE - 1))
  256. goto out;
  257. if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
  258. goto out;
  259. if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
  260. goto out;
  261. memslot = &kvm->memslots[mem->slot];
  262. base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
  263. npages = mem->memory_size >> PAGE_SHIFT;
  264. if (!npages)
  265. mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
  266. new = old = *memslot;
  267. new.base_gfn = base_gfn;
  268. new.npages = npages;
  269. new.flags = mem->flags;
  270. /* Disallow changing a memory slot's size. */
  271. r = -EINVAL;
  272. if (npages && old.npages && npages != old.npages)
  273. goto out_free;
  274. /* Check for overlaps */
  275. r = -EEXIST;
  276. for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
  277. struct kvm_memory_slot *s = &kvm->memslots[i];
  278. if (s == memslot)
  279. continue;
  280. if (!((base_gfn + npages <= s->base_gfn) ||
  281. (base_gfn >= s->base_gfn + s->npages)))
  282. goto out_free;
  283. }
  284. /* Free page dirty bitmap if unneeded */
  285. if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
  286. new.dirty_bitmap = NULL;
  287. r = -ENOMEM;
  288. /* Allocate if a slot is being created */
  289. if (npages && !new.rmap) {
  290. new.rmap = vmalloc(npages * sizeof(struct page *));
  291. if (!new.rmap)
  292. goto out_free;
  293. memset(new.rmap, 0, npages * sizeof(*new.rmap));
  294. new.user_alloc = user_alloc;
  295. if (user_alloc)
  296. new.userspace_addr = mem->userspace_addr;
  297. else {
  298. down_write(&current->mm->mmap_sem);
  299. new.userspace_addr = do_mmap(NULL, 0,
  300. npages * PAGE_SIZE,
  301. PROT_READ | PROT_WRITE,
  302. MAP_SHARED | MAP_ANONYMOUS,
  303. 0);
  304. up_write(&current->mm->mmap_sem);
  305. if (IS_ERR((void *)new.userspace_addr))
  306. goto out_free;
  307. }
  308. } else {
  309. if (!old.user_alloc && old.rmap) {
  310. int ret;
  311. down_write(&current->mm->mmap_sem);
  312. ret = do_munmap(current->mm, old.userspace_addr,
  313. old.npages * PAGE_SIZE);
  314. up_write(&current->mm->mmap_sem);
  315. if (ret < 0)
  316. printk(KERN_WARNING
  317. "kvm_vm_ioctl_set_memory_region: "
  318. "failed to munmap memory\n");
  319. }
  320. }
  321. /* Allocate page dirty bitmap if needed */
  322. if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
  323. unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
  324. new.dirty_bitmap = vmalloc(dirty_bytes);
  325. if (!new.dirty_bitmap)
  326. goto out_free;
  327. memset(new.dirty_bitmap, 0, dirty_bytes);
  328. }
  329. if (mem->slot >= kvm->nmemslots)
  330. kvm->nmemslots = mem->slot + 1;
  331. if (!kvm->n_requested_mmu_pages) {
  332. unsigned int n_pages;
  333. if (npages) {
  334. n_pages = npages * KVM_PERMILLE_MMU_PAGES / 1000;
  335. kvm_mmu_change_mmu_pages(kvm, kvm->n_alloc_mmu_pages +
  336. n_pages);
  337. } else {
  338. unsigned int nr_mmu_pages;
  339. n_pages = old.npages * KVM_PERMILLE_MMU_PAGES / 1000;
  340. nr_mmu_pages = kvm->n_alloc_mmu_pages - n_pages;
  341. nr_mmu_pages = max(nr_mmu_pages,
  342. (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
  343. kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
  344. }
  345. }
  346. *memslot = new;
  347. kvm_mmu_slot_remove_write_access(kvm, mem->slot);
  348. kvm_flush_remote_tlbs(kvm);
  349. kvm_free_physmem_slot(&old, &new);
  350. return 0;
  351. out_free:
  352. kvm_free_physmem_slot(&new, &old);
  353. out:
  354. return r;
  355. }
  356. EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
  357. int kvm_set_memory_region(struct kvm *kvm,
  358. struct kvm_userspace_memory_region *mem,
  359. int user_alloc)
  360. {
  361. int r;
  362. mutex_lock(&kvm->lock);
  363. r = __kvm_set_memory_region(kvm, mem, user_alloc);
  364. mutex_unlock(&kvm->lock);
  365. return r;
  366. }
  367. EXPORT_SYMBOL_GPL(kvm_set_memory_region);
  368. int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  369. struct
  370. kvm_userspace_memory_region *mem,
  371. int user_alloc)
  372. {
  373. if (mem->slot >= KVM_MEMORY_SLOTS)
  374. return -EINVAL;
  375. return kvm_set_memory_region(kvm, mem, user_alloc);
  376. }
  377. /*
  378. * Get (and clear) the dirty memory log for a memory slot.
  379. */
  380. static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
  381. struct kvm_dirty_log *log)
  382. {
  383. struct kvm_memory_slot *memslot;
  384. int r, i;
  385. int n;
  386. unsigned long any = 0;
  387. mutex_lock(&kvm->lock);
  388. r = -EINVAL;
  389. if (log->slot >= KVM_MEMORY_SLOTS)
  390. goto out;
  391. memslot = &kvm->memslots[log->slot];
  392. r = -ENOENT;
  393. if (!memslot->dirty_bitmap)
  394. goto out;
  395. n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
  396. for (i = 0; !any && i < n/sizeof(long); ++i)
  397. any = memslot->dirty_bitmap[i];
  398. r = -EFAULT;
  399. if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
  400. goto out;
  401. /* If nothing is dirty, don't bother messing with page tables. */
  402. if (any) {
  403. kvm_mmu_slot_remove_write_access(kvm, log->slot);
  404. kvm_flush_remote_tlbs(kvm);
  405. memset(memslot->dirty_bitmap, 0, n);
  406. }
  407. r = 0;
  408. out:
  409. mutex_unlock(&kvm->lock);
  410. return r;
  411. }
  412. int is_error_page(struct page *page)
  413. {
  414. return page == bad_page;
  415. }
  416. EXPORT_SYMBOL_GPL(is_error_page);
  417. gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
  418. {
  419. int i;
  420. struct kvm_mem_alias *alias;
  421. for (i = 0; i < kvm->naliases; ++i) {
  422. alias = &kvm->aliases[i];
  423. if (gfn >= alias->base_gfn
  424. && gfn < alias->base_gfn + alias->npages)
  425. return alias->target_gfn + gfn - alias->base_gfn;
  426. }
  427. return gfn;
  428. }
  429. static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  430. {
  431. int i;
  432. for (i = 0; i < kvm->nmemslots; ++i) {
  433. struct kvm_memory_slot *memslot = &kvm->memslots[i];
  434. if (gfn >= memslot->base_gfn
  435. && gfn < memslot->base_gfn + memslot->npages)
  436. return memslot;
  437. }
  438. return NULL;
  439. }
  440. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  441. {
  442. gfn = unalias_gfn(kvm, gfn);
  443. return __gfn_to_memslot(kvm, gfn);
  444. }
  445. int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
  446. {
  447. int i;
  448. gfn = unalias_gfn(kvm, gfn);
  449. for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
  450. struct kvm_memory_slot *memslot = &kvm->memslots[i];
  451. if (gfn >= memslot->base_gfn
  452. && gfn < memslot->base_gfn + memslot->npages)
  453. return 1;
  454. }
  455. return 0;
  456. }
  457. EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
  458. /*
  459. * Requires current->mm->mmap_sem to be held
  460. */
  461. static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn)
  462. {
  463. struct kvm_memory_slot *slot;
  464. struct page *page[1];
  465. int npages;
  466. might_sleep();
  467. gfn = unalias_gfn(kvm, gfn);
  468. slot = __gfn_to_memslot(kvm, gfn);
  469. if (!slot) {
  470. get_page(bad_page);
  471. return bad_page;
  472. }
  473. npages = get_user_pages(current, current->mm,
  474. slot->userspace_addr
  475. + (gfn - slot->base_gfn) * PAGE_SIZE, 1,
  476. 1, 1, page, NULL);
  477. if (npages != 1) {
  478. get_page(bad_page);
  479. return bad_page;
  480. }
  481. return page[0];
  482. }
  483. struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
  484. {
  485. struct page *page;
  486. down_read(&current->mm->mmap_sem);
  487. page = __gfn_to_page(kvm, gfn);
  488. up_read(&current->mm->mmap_sem);
  489. return page;
  490. }
  491. EXPORT_SYMBOL_GPL(gfn_to_page);
  492. void kvm_release_page(struct page *page)
  493. {
  494. if (!PageReserved(page))
  495. SetPageDirty(page);
  496. put_page(page);
  497. }
  498. EXPORT_SYMBOL_GPL(kvm_release_page);
  499. static int next_segment(unsigned long len, int offset)
  500. {
  501. if (len > PAGE_SIZE - offset)
  502. return PAGE_SIZE - offset;
  503. else
  504. return len;
  505. }
  506. int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
  507. int len)
  508. {
  509. void *page_virt;
  510. struct page *page;
  511. page = gfn_to_page(kvm, gfn);
  512. if (is_error_page(page)) {
  513. kvm_release_page(page);
  514. return -EFAULT;
  515. }
  516. page_virt = kmap_atomic(page, KM_USER0);
  517. memcpy(data, page_virt + offset, len);
  518. kunmap_atomic(page_virt, KM_USER0);
  519. kvm_release_page(page);
  520. return 0;
  521. }
  522. EXPORT_SYMBOL_GPL(kvm_read_guest_page);
  523. int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
  524. {
  525. gfn_t gfn = gpa >> PAGE_SHIFT;
  526. int seg;
  527. int offset = offset_in_page(gpa);
  528. int ret;
  529. while ((seg = next_segment(len, offset)) != 0) {
  530. ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
  531. if (ret < 0)
  532. return ret;
  533. offset = 0;
  534. len -= seg;
  535. data += seg;
  536. ++gfn;
  537. }
  538. return 0;
  539. }
  540. EXPORT_SYMBOL_GPL(kvm_read_guest);
  541. int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
  542. int offset, int len)
  543. {
  544. void *page_virt;
  545. struct page *page;
  546. page = gfn_to_page(kvm, gfn);
  547. if (is_error_page(page)) {
  548. kvm_release_page(page);
  549. return -EFAULT;
  550. }
  551. page_virt = kmap_atomic(page, KM_USER0);
  552. memcpy(page_virt + offset, data, len);
  553. kunmap_atomic(page_virt, KM_USER0);
  554. mark_page_dirty(kvm, gfn);
  555. kvm_release_page(page);
  556. return 0;
  557. }
  558. EXPORT_SYMBOL_GPL(kvm_write_guest_page);
  559. int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
  560. unsigned long len)
  561. {
  562. gfn_t gfn = gpa >> PAGE_SHIFT;
  563. int seg;
  564. int offset = offset_in_page(gpa);
  565. int ret;
  566. while ((seg = next_segment(len, offset)) != 0) {
  567. ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
  568. if (ret < 0)
  569. return ret;
  570. offset = 0;
  571. len -= seg;
  572. data += seg;
  573. ++gfn;
  574. }
  575. return 0;
  576. }
  577. int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
  578. {
  579. void *page_virt;
  580. struct page *page;
  581. page = gfn_to_page(kvm, gfn);
  582. if (is_error_page(page)) {
  583. kvm_release_page(page);
  584. return -EFAULT;
  585. }
  586. page_virt = kmap_atomic(page, KM_USER0);
  587. memset(page_virt + offset, 0, len);
  588. kunmap_atomic(page_virt, KM_USER0);
  589. kvm_release_page(page);
  590. return 0;
  591. }
  592. EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
  593. int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
  594. {
  595. gfn_t gfn = gpa >> PAGE_SHIFT;
  596. int seg;
  597. int offset = offset_in_page(gpa);
  598. int ret;
  599. while ((seg = next_segment(len, offset)) != 0) {
  600. ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
  601. if (ret < 0)
  602. return ret;
  603. offset = 0;
  604. len -= seg;
  605. ++gfn;
  606. }
  607. return 0;
  608. }
  609. EXPORT_SYMBOL_GPL(kvm_clear_guest);
  610. void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
  611. {
  612. struct kvm_memory_slot *memslot;
  613. gfn = unalias_gfn(kvm, gfn);
  614. memslot = __gfn_to_memslot(kvm, gfn);
  615. if (memslot && memslot->dirty_bitmap) {
  616. unsigned long rel_gfn = gfn - memslot->base_gfn;
  617. /* avoid RMW */
  618. if (!test_bit(rel_gfn, memslot->dirty_bitmap))
  619. set_bit(rel_gfn, memslot->dirty_bitmap);
  620. }
  621. }
  622. /*
  623. * The vCPU has executed a HLT instruction with in-kernel mode enabled.
  624. */
  625. void kvm_vcpu_block(struct kvm_vcpu *vcpu)
  626. {
  627. DECLARE_WAITQUEUE(wait, current);
  628. add_wait_queue(&vcpu->wq, &wait);
  629. /*
  630. * We will block until either an interrupt or a signal wakes us up
  631. */
  632. while (!kvm_cpu_has_interrupt(vcpu)
  633. && !signal_pending(current)
  634. && vcpu->mp_state != VCPU_MP_STATE_RUNNABLE
  635. && vcpu->mp_state != VCPU_MP_STATE_SIPI_RECEIVED) {
  636. set_current_state(TASK_INTERRUPTIBLE);
  637. vcpu_put(vcpu);
  638. schedule();
  639. vcpu_load(vcpu);
  640. }
  641. __set_current_state(TASK_RUNNING);
  642. remove_wait_queue(&vcpu->wq, &wait);
  643. }
  644. void kvm_resched(struct kvm_vcpu *vcpu)
  645. {
  646. if (!need_resched())
  647. return;
  648. cond_resched();
  649. }
  650. EXPORT_SYMBOL_GPL(kvm_resched);
  651. /*
  652. * Translate a guest virtual address to a guest physical address.
  653. */
  654. static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  655. struct kvm_translation *tr)
  656. {
  657. unsigned long vaddr = tr->linear_address;
  658. gpa_t gpa;
  659. vcpu_load(vcpu);
  660. mutex_lock(&vcpu->kvm->lock);
  661. gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
  662. tr->physical_address = gpa;
  663. tr->valid = gpa != UNMAPPED_GVA;
  664. tr->writeable = 1;
  665. tr->usermode = 0;
  666. mutex_unlock(&vcpu->kvm->lock);
  667. vcpu_put(vcpu);
  668. return 0;
  669. }
  670. static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
  671. struct kvm_interrupt *irq)
  672. {
  673. if (irq->irq < 0 || irq->irq >= 256)
  674. return -EINVAL;
  675. if (irqchip_in_kernel(vcpu->kvm))
  676. return -ENXIO;
  677. vcpu_load(vcpu);
  678. set_bit(irq->irq, vcpu->irq_pending);
  679. set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
  680. vcpu_put(vcpu);
  681. return 0;
  682. }
  683. static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
  684. unsigned long address,
  685. int *type)
  686. {
  687. struct kvm_vcpu *vcpu = vma->vm_file->private_data;
  688. unsigned long pgoff;
  689. struct page *page;
  690. pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  691. if (pgoff == 0)
  692. page = virt_to_page(vcpu->run);
  693. else if (pgoff == KVM_PIO_PAGE_OFFSET)
  694. page = virt_to_page(vcpu->pio_data);
  695. else
  696. return NOPAGE_SIGBUS;
  697. get_page(page);
  698. if (type != NULL)
  699. *type = VM_FAULT_MINOR;
  700. return page;
  701. }
  702. static struct vm_operations_struct kvm_vcpu_vm_ops = {
  703. .nopage = kvm_vcpu_nopage,
  704. };
  705. static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
  706. {
  707. vma->vm_ops = &kvm_vcpu_vm_ops;
  708. return 0;
  709. }
  710. static int kvm_vcpu_release(struct inode *inode, struct file *filp)
  711. {
  712. struct kvm_vcpu *vcpu = filp->private_data;
  713. fput(vcpu->kvm->filp);
  714. return 0;
  715. }
  716. static struct file_operations kvm_vcpu_fops = {
  717. .release = kvm_vcpu_release,
  718. .unlocked_ioctl = kvm_vcpu_ioctl,
  719. .compat_ioctl = kvm_vcpu_ioctl,
  720. .mmap = kvm_vcpu_mmap,
  721. };
  722. /*
  723. * Allocates an inode for the vcpu.
  724. */
  725. static int create_vcpu_fd(struct kvm_vcpu *vcpu)
  726. {
  727. int fd, r;
  728. struct inode *inode;
  729. struct file *file;
  730. r = anon_inode_getfd(&fd, &inode, &file,
  731. "kvm-vcpu", &kvm_vcpu_fops, vcpu);
  732. if (r)
  733. return r;
  734. atomic_inc(&vcpu->kvm->filp->f_count);
  735. return fd;
  736. }
  737. /*
  738. * Creates some virtual cpus. Good luck creating more than one.
  739. */
  740. static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
  741. {
  742. int r;
  743. struct kvm_vcpu *vcpu;
  744. if (!valid_vcpu(n))
  745. return -EINVAL;
  746. vcpu = kvm_x86_ops->vcpu_create(kvm, n);
  747. if (IS_ERR(vcpu))
  748. return PTR_ERR(vcpu);
  749. preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
  750. /* We do fxsave: this must be aligned. */
  751. BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF);
  752. vcpu_load(vcpu);
  753. r = kvm_x86_ops->vcpu_reset(vcpu);
  754. if (r == 0)
  755. r = kvm_mmu_setup(vcpu);
  756. vcpu_put(vcpu);
  757. if (r < 0)
  758. goto free_vcpu;
  759. mutex_lock(&kvm->lock);
  760. if (kvm->vcpus[n]) {
  761. r = -EEXIST;
  762. mutex_unlock(&kvm->lock);
  763. goto mmu_unload;
  764. }
  765. kvm->vcpus[n] = vcpu;
  766. mutex_unlock(&kvm->lock);
  767. /* Now it's all set up, let userspace reach it */
  768. r = create_vcpu_fd(vcpu);
  769. if (r < 0)
  770. goto unlink;
  771. return r;
  772. unlink:
  773. mutex_lock(&kvm->lock);
  774. kvm->vcpus[n] = NULL;
  775. mutex_unlock(&kvm->lock);
  776. mmu_unload:
  777. vcpu_load(vcpu);
  778. kvm_mmu_unload(vcpu);
  779. vcpu_put(vcpu);
  780. free_vcpu:
  781. kvm_x86_ops->vcpu_free(vcpu);
  782. return r;
  783. }
  784. static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
  785. {
  786. if (sigset) {
  787. sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
  788. vcpu->sigset_active = 1;
  789. vcpu->sigset = *sigset;
  790. } else
  791. vcpu->sigset_active = 0;
  792. return 0;
  793. }
  794. static long kvm_vcpu_ioctl(struct file *filp,
  795. unsigned int ioctl, unsigned long arg)
  796. {
  797. struct kvm_vcpu *vcpu = filp->private_data;
  798. void __user *argp = (void __user *)arg;
  799. int r;
  800. switch (ioctl) {
  801. case KVM_RUN:
  802. r = -EINVAL;
  803. if (arg)
  804. goto out;
  805. r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
  806. break;
  807. case KVM_GET_REGS: {
  808. struct kvm_regs kvm_regs;
  809. memset(&kvm_regs, 0, sizeof kvm_regs);
  810. r = kvm_arch_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
  811. if (r)
  812. goto out;
  813. r = -EFAULT;
  814. if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
  815. goto out;
  816. r = 0;
  817. break;
  818. }
  819. case KVM_SET_REGS: {
  820. struct kvm_regs kvm_regs;
  821. r = -EFAULT;
  822. if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
  823. goto out;
  824. r = kvm_arch_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
  825. if (r)
  826. goto out;
  827. r = 0;
  828. break;
  829. }
  830. case KVM_GET_SREGS: {
  831. struct kvm_sregs kvm_sregs;
  832. memset(&kvm_sregs, 0, sizeof kvm_sregs);
  833. r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
  834. if (r)
  835. goto out;
  836. r = -EFAULT;
  837. if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
  838. goto out;
  839. r = 0;
  840. break;
  841. }
  842. case KVM_SET_SREGS: {
  843. struct kvm_sregs kvm_sregs;
  844. r = -EFAULT;
  845. if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
  846. goto out;
  847. r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
  848. if (r)
  849. goto out;
  850. r = 0;
  851. break;
  852. }
  853. case KVM_TRANSLATE: {
  854. struct kvm_translation tr;
  855. r = -EFAULT;
  856. if (copy_from_user(&tr, argp, sizeof tr))
  857. goto out;
  858. r = kvm_vcpu_ioctl_translate(vcpu, &tr);
  859. if (r)
  860. goto out;
  861. r = -EFAULT;
  862. if (copy_to_user(argp, &tr, sizeof tr))
  863. goto out;
  864. r = 0;
  865. break;
  866. }
  867. case KVM_INTERRUPT: {
  868. struct kvm_interrupt irq;
  869. r = -EFAULT;
  870. if (copy_from_user(&irq, argp, sizeof irq))
  871. goto out;
  872. r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
  873. if (r)
  874. goto out;
  875. r = 0;
  876. break;
  877. }
  878. case KVM_DEBUG_GUEST: {
  879. struct kvm_debug_guest dbg;
  880. r = -EFAULT;
  881. if (copy_from_user(&dbg, argp, sizeof dbg))
  882. goto out;
  883. r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
  884. if (r)
  885. goto out;
  886. r = 0;
  887. break;
  888. }
  889. case KVM_SET_SIGNAL_MASK: {
  890. struct kvm_signal_mask __user *sigmask_arg = argp;
  891. struct kvm_signal_mask kvm_sigmask;
  892. sigset_t sigset, *p;
  893. p = NULL;
  894. if (argp) {
  895. r = -EFAULT;
  896. if (copy_from_user(&kvm_sigmask, argp,
  897. sizeof kvm_sigmask))
  898. goto out;
  899. r = -EINVAL;
  900. if (kvm_sigmask.len != sizeof sigset)
  901. goto out;
  902. r = -EFAULT;
  903. if (copy_from_user(&sigset, sigmask_arg->sigset,
  904. sizeof sigset))
  905. goto out;
  906. p = &sigset;
  907. }
  908. r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
  909. break;
  910. }
  911. case KVM_GET_FPU: {
  912. struct kvm_fpu fpu;
  913. memset(&fpu, 0, sizeof fpu);
  914. r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu);
  915. if (r)
  916. goto out;
  917. r = -EFAULT;
  918. if (copy_to_user(argp, &fpu, sizeof fpu))
  919. goto out;
  920. r = 0;
  921. break;
  922. }
  923. case KVM_SET_FPU: {
  924. struct kvm_fpu fpu;
  925. r = -EFAULT;
  926. if (copy_from_user(&fpu, argp, sizeof fpu))
  927. goto out;
  928. r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu);
  929. if (r)
  930. goto out;
  931. r = 0;
  932. break;
  933. }
  934. default:
  935. r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  936. }
  937. out:
  938. return r;
  939. }
  940. static long kvm_vm_ioctl(struct file *filp,
  941. unsigned int ioctl, unsigned long arg)
  942. {
  943. struct kvm *kvm = filp->private_data;
  944. void __user *argp = (void __user *)arg;
  945. int r;
  946. switch (ioctl) {
  947. case KVM_CREATE_VCPU:
  948. r = kvm_vm_ioctl_create_vcpu(kvm, arg);
  949. if (r < 0)
  950. goto out;
  951. break;
  952. case KVM_SET_USER_MEMORY_REGION: {
  953. struct kvm_userspace_memory_region kvm_userspace_mem;
  954. r = -EFAULT;
  955. if (copy_from_user(&kvm_userspace_mem, argp,
  956. sizeof kvm_userspace_mem))
  957. goto out;
  958. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
  959. if (r)
  960. goto out;
  961. break;
  962. }
  963. case KVM_GET_DIRTY_LOG: {
  964. struct kvm_dirty_log log;
  965. r = -EFAULT;
  966. if (copy_from_user(&log, argp, sizeof log))
  967. goto out;
  968. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  969. if (r)
  970. goto out;
  971. break;
  972. }
  973. default:
  974. r = kvm_arch_vm_ioctl(filp, ioctl, arg);
  975. }
  976. out:
  977. return r;
  978. }
  979. static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
  980. unsigned long address,
  981. int *type)
  982. {
  983. struct kvm *kvm = vma->vm_file->private_data;
  984. unsigned long pgoff;
  985. struct page *page;
  986. pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  987. if (!kvm_is_visible_gfn(kvm, pgoff))
  988. return NOPAGE_SIGBUS;
  989. /* current->mm->mmap_sem is already held so call lockless version */
  990. page = __gfn_to_page(kvm, pgoff);
  991. if (is_error_page(page)) {
  992. kvm_release_page(page);
  993. return NOPAGE_SIGBUS;
  994. }
  995. if (type != NULL)
  996. *type = VM_FAULT_MINOR;
  997. return page;
  998. }
  999. static struct vm_operations_struct kvm_vm_vm_ops = {
  1000. .nopage = kvm_vm_nopage,
  1001. };
  1002. static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
  1003. {
  1004. vma->vm_ops = &kvm_vm_vm_ops;
  1005. return 0;
  1006. }
  1007. static struct file_operations kvm_vm_fops = {
  1008. .release = kvm_vm_release,
  1009. .unlocked_ioctl = kvm_vm_ioctl,
  1010. .compat_ioctl = kvm_vm_ioctl,
  1011. .mmap = kvm_vm_mmap,
  1012. };
  1013. static int kvm_dev_ioctl_create_vm(void)
  1014. {
  1015. int fd, r;
  1016. struct inode *inode;
  1017. struct file *file;
  1018. struct kvm *kvm;
  1019. kvm = kvm_create_vm();
  1020. if (IS_ERR(kvm))
  1021. return PTR_ERR(kvm);
  1022. r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
  1023. if (r) {
  1024. kvm_destroy_vm(kvm);
  1025. return r;
  1026. }
  1027. kvm->filp = file;
  1028. return fd;
  1029. }
  1030. static long kvm_dev_ioctl(struct file *filp,
  1031. unsigned int ioctl, unsigned long arg)
  1032. {
  1033. void __user *argp = (void __user *)arg;
  1034. long r = -EINVAL;
  1035. switch (ioctl) {
  1036. case KVM_GET_API_VERSION:
  1037. r = -EINVAL;
  1038. if (arg)
  1039. goto out;
  1040. r = KVM_API_VERSION;
  1041. break;
  1042. case KVM_CREATE_VM:
  1043. r = -EINVAL;
  1044. if (arg)
  1045. goto out;
  1046. r = kvm_dev_ioctl_create_vm();
  1047. break;
  1048. case KVM_CHECK_EXTENSION: {
  1049. int ext = (long)argp;
  1050. switch (ext) {
  1051. case KVM_CAP_IRQCHIP:
  1052. case KVM_CAP_HLT:
  1053. case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
  1054. case KVM_CAP_USER_MEMORY:
  1055. case KVM_CAP_SET_TSS_ADDR:
  1056. r = 1;
  1057. break;
  1058. default:
  1059. r = 0;
  1060. break;
  1061. }
  1062. break;
  1063. }
  1064. case KVM_GET_VCPU_MMAP_SIZE:
  1065. r = -EINVAL;
  1066. if (arg)
  1067. goto out;
  1068. r = 2 * PAGE_SIZE;
  1069. break;
  1070. default:
  1071. return kvm_arch_dev_ioctl(filp, ioctl, arg);
  1072. }
  1073. out:
  1074. return r;
  1075. }
  1076. static struct file_operations kvm_chardev_ops = {
  1077. .unlocked_ioctl = kvm_dev_ioctl,
  1078. .compat_ioctl = kvm_dev_ioctl,
  1079. };
  1080. static struct miscdevice kvm_dev = {
  1081. KVM_MINOR,
  1082. "kvm",
  1083. &kvm_chardev_ops,
  1084. };
  1085. /*
  1086. * Make sure that a cpu that is being hot-unplugged does not have any vcpus
  1087. * cached on it.
  1088. */
  1089. static void decache_vcpus_on_cpu(int cpu)
  1090. {
  1091. struct kvm *vm;
  1092. struct kvm_vcpu *vcpu;
  1093. int i;
  1094. spin_lock(&kvm_lock);
  1095. list_for_each_entry(vm, &vm_list, vm_list)
  1096. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  1097. vcpu = vm->vcpus[i];
  1098. if (!vcpu)
  1099. continue;
  1100. /*
  1101. * If the vcpu is locked, then it is running on some
  1102. * other cpu and therefore it is not cached on the
  1103. * cpu in question.
  1104. *
  1105. * If it's not locked, check the last cpu it executed
  1106. * on.
  1107. */
  1108. if (mutex_trylock(&vcpu->mutex)) {
  1109. if (vcpu->cpu == cpu) {
  1110. kvm_x86_ops->vcpu_decache(vcpu);
  1111. vcpu->cpu = -1;
  1112. }
  1113. mutex_unlock(&vcpu->mutex);
  1114. }
  1115. }
  1116. spin_unlock(&kvm_lock);
  1117. }
  1118. static void hardware_enable(void *junk)
  1119. {
  1120. int cpu = raw_smp_processor_id();
  1121. if (cpu_isset(cpu, cpus_hardware_enabled))
  1122. return;
  1123. cpu_set(cpu, cpus_hardware_enabled);
  1124. kvm_x86_ops->hardware_enable(NULL);
  1125. }
  1126. static void hardware_disable(void *junk)
  1127. {
  1128. int cpu = raw_smp_processor_id();
  1129. if (!cpu_isset(cpu, cpus_hardware_enabled))
  1130. return;
  1131. cpu_clear(cpu, cpus_hardware_enabled);
  1132. decache_vcpus_on_cpu(cpu);
  1133. kvm_x86_ops->hardware_disable(NULL);
  1134. }
  1135. static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  1136. void *v)
  1137. {
  1138. int cpu = (long)v;
  1139. switch (val) {
  1140. case CPU_DYING:
  1141. case CPU_DYING_FROZEN:
  1142. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  1143. cpu);
  1144. hardware_disable(NULL);
  1145. break;
  1146. case CPU_UP_CANCELED:
  1147. case CPU_UP_CANCELED_FROZEN:
  1148. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  1149. cpu);
  1150. smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
  1151. break;
  1152. case CPU_ONLINE:
  1153. case CPU_ONLINE_FROZEN:
  1154. printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
  1155. cpu);
  1156. smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
  1157. break;
  1158. }
  1159. return NOTIFY_OK;
  1160. }
  1161. static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
  1162. void *v)
  1163. {
  1164. if (val == SYS_RESTART) {
  1165. /*
  1166. * Some (well, at least mine) BIOSes hang on reboot if
  1167. * in vmx root mode.
  1168. */
  1169. printk(KERN_INFO "kvm: exiting hardware virtualization\n");
  1170. on_each_cpu(hardware_disable, NULL, 0, 1);
  1171. }
  1172. return NOTIFY_OK;
  1173. }
  1174. static struct notifier_block kvm_reboot_notifier = {
  1175. .notifier_call = kvm_reboot,
  1176. .priority = 0,
  1177. };
  1178. void kvm_io_bus_init(struct kvm_io_bus *bus)
  1179. {
  1180. memset(bus, 0, sizeof(*bus));
  1181. }
  1182. void kvm_io_bus_destroy(struct kvm_io_bus *bus)
  1183. {
  1184. int i;
  1185. for (i = 0; i < bus->dev_count; i++) {
  1186. struct kvm_io_device *pos = bus->devs[i];
  1187. kvm_iodevice_destructor(pos);
  1188. }
  1189. }
  1190. struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
  1191. {
  1192. int i;
  1193. for (i = 0; i < bus->dev_count; i++) {
  1194. struct kvm_io_device *pos = bus->devs[i];
  1195. if (pos->in_range(pos, addr))
  1196. return pos;
  1197. }
  1198. return NULL;
  1199. }
  1200. void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
  1201. {
  1202. BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
  1203. bus->devs[bus->dev_count++] = dev;
  1204. }
  1205. static struct notifier_block kvm_cpu_notifier = {
  1206. .notifier_call = kvm_cpu_hotplug,
  1207. .priority = 20, /* must be > scheduler priority */
  1208. };
  1209. static u64 stat_get(void *_offset)
  1210. {
  1211. unsigned offset = (long)_offset;
  1212. u64 total = 0;
  1213. struct kvm *kvm;
  1214. struct kvm_vcpu *vcpu;
  1215. int i;
  1216. spin_lock(&kvm_lock);
  1217. list_for_each_entry(kvm, &vm_list, vm_list)
  1218. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  1219. vcpu = kvm->vcpus[i];
  1220. if (vcpu)
  1221. total += *(u32 *)((void *)vcpu + offset);
  1222. }
  1223. spin_unlock(&kvm_lock);
  1224. return total;
  1225. }
  1226. DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n");
  1227. static __init void kvm_init_debug(void)
  1228. {
  1229. struct kvm_stats_debugfs_item *p;
  1230. debugfs_dir = debugfs_create_dir("kvm", NULL);
  1231. for (p = debugfs_entries; p->name; ++p)
  1232. p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
  1233. (void *)(long)p->offset,
  1234. &stat_fops);
  1235. }
  1236. static void kvm_exit_debug(void)
  1237. {
  1238. struct kvm_stats_debugfs_item *p;
  1239. for (p = debugfs_entries; p->name; ++p)
  1240. debugfs_remove(p->dentry);
  1241. debugfs_remove(debugfs_dir);
  1242. }
  1243. static int kvm_suspend(struct sys_device *dev, pm_message_t state)
  1244. {
  1245. hardware_disable(NULL);
  1246. return 0;
  1247. }
  1248. static int kvm_resume(struct sys_device *dev)
  1249. {
  1250. hardware_enable(NULL);
  1251. return 0;
  1252. }
  1253. static struct sysdev_class kvm_sysdev_class = {
  1254. .name = "kvm",
  1255. .suspend = kvm_suspend,
  1256. .resume = kvm_resume,
  1257. };
  1258. static struct sys_device kvm_sysdev = {
  1259. .id = 0,
  1260. .cls = &kvm_sysdev_class,
  1261. };
  1262. struct page *bad_page;
  1263. static inline
  1264. struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
  1265. {
  1266. return container_of(pn, struct kvm_vcpu, preempt_notifier);
  1267. }
  1268. static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
  1269. {
  1270. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  1271. kvm_x86_ops->vcpu_load(vcpu, cpu);
  1272. }
  1273. static void kvm_sched_out(struct preempt_notifier *pn,
  1274. struct task_struct *next)
  1275. {
  1276. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  1277. kvm_x86_ops->vcpu_put(vcpu);
  1278. }
  1279. int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size,
  1280. struct module *module)
  1281. {
  1282. int r;
  1283. int cpu;
  1284. if (kvm_x86_ops) {
  1285. printk(KERN_ERR "kvm: already loaded the other module\n");
  1286. return -EEXIST;
  1287. }
  1288. if (!ops->cpu_has_kvm_support()) {
  1289. printk(KERN_ERR "kvm: no hardware support\n");
  1290. return -EOPNOTSUPP;
  1291. }
  1292. if (ops->disabled_by_bios()) {
  1293. printk(KERN_ERR "kvm: disabled by bios\n");
  1294. return -EOPNOTSUPP;
  1295. }
  1296. kvm_x86_ops = ops;
  1297. r = kvm_x86_ops->hardware_setup();
  1298. if (r < 0)
  1299. goto out;
  1300. for_each_online_cpu(cpu) {
  1301. smp_call_function_single(cpu,
  1302. kvm_x86_ops->check_processor_compatibility,
  1303. &r, 0, 1);
  1304. if (r < 0)
  1305. goto out_free_0;
  1306. }
  1307. on_each_cpu(hardware_enable, NULL, 0, 1);
  1308. r = register_cpu_notifier(&kvm_cpu_notifier);
  1309. if (r)
  1310. goto out_free_1;
  1311. register_reboot_notifier(&kvm_reboot_notifier);
  1312. r = sysdev_class_register(&kvm_sysdev_class);
  1313. if (r)
  1314. goto out_free_2;
  1315. r = sysdev_register(&kvm_sysdev);
  1316. if (r)
  1317. goto out_free_3;
  1318. /* A kmem cache lets us meet the alignment requirements of fx_save. */
  1319. kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
  1320. __alignof__(struct kvm_vcpu), 0, 0);
  1321. if (!kvm_vcpu_cache) {
  1322. r = -ENOMEM;
  1323. goto out_free_4;
  1324. }
  1325. kvm_chardev_ops.owner = module;
  1326. r = misc_register(&kvm_dev);
  1327. if (r) {
  1328. printk(KERN_ERR "kvm: misc device register failed\n");
  1329. goto out_free;
  1330. }
  1331. kvm_preempt_ops.sched_in = kvm_sched_in;
  1332. kvm_preempt_ops.sched_out = kvm_sched_out;
  1333. kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
  1334. return 0;
  1335. out_free:
  1336. kmem_cache_destroy(kvm_vcpu_cache);
  1337. out_free_4:
  1338. sysdev_unregister(&kvm_sysdev);
  1339. out_free_3:
  1340. sysdev_class_unregister(&kvm_sysdev_class);
  1341. out_free_2:
  1342. unregister_reboot_notifier(&kvm_reboot_notifier);
  1343. unregister_cpu_notifier(&kvm_cpu_notifier);
  1344. out_free_1:
  1345. on_each_cpu(hardware_disable, NULL, 0, 1);
  1346. out_free_0:
  1347. kvm_x86_ops->hardware_unsetup();
  1348. out:
  1349. kvm_x86_ops = NULL;
  1350. return r;
  1351. }
  1352. EXPORT_SYMBOL_GPL(kvm_init_x86);
  1353. void kvm_exit_x86(void)
  1354. {
  1355. misc_deregister(&kvm_dev);
  1356. kmem_cache_destroy(kvm_vcpu_cache);
  1357. sysdev_unregister(&kvm_sysdev);
  1358. sysdev_class_unregister(&kvm_sysdev_class);
  1359. unregister_reboot_notifier(&kvm_reboot_notifier);
  1360. unregister_cpu_notifier(&kvm_cpu_notifier);
  1361. on_each_cpu(hardware_disable, NULL, 0, 1);
  1362. kvm_x86_ops->hardware_unsetup();
  1363. kvm_x86_ops = NULL;
  1364. }
  1365. EXPORT_SYMBOL_GPL(kvm_exit_x86);
  1366. static __init int kvm_init(void)
  1367. {
  1368. int r;
  1369. r = kvm_mmu_module_init();
  1370. if (r)
  1371. goto out4;
  1372. kvm_init_debug();
  1373. kvm_arch_init();
  1374. bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  1375. if (bad_page == NULL) {
  1376. r = -ENOMEM;
  1377. goto out;
  1378. }
  1379. return 0;
  1380. out:
  1381. kvm_exit_debug();
  1382. kvm_mmu_module_exit();
  1383. out4:
  1384. return r;
  1385. }
  1386. static __exit void kvm_exit(void)
  1387. {
  1388. kvm_exit_debug();
  1389. __free_page(bad_page);
  1390. kvm_mmu_module_exit();
  1391. }
  1392. module_init(kvm_init)
  1393. module_exit(kvm_exit)