kvm_main.c 28 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. *
  9. * Authors:
  10. * Avi Kivity <avi@qumranet.com>
  11. * Yaniv Kamay <yaniv@qumranet.com>
  12. *
  13. * This work is licensed under the terms of the GNU GPL, version 2. See
  14. * the COPYING file in the top-level directory.
  15. *
  16. */
  17. #include "iodev.h"
  18. #include <linux/kvm_host.h>
  19. #include <linux/kvm.h>
  20. #include <linux/module.h>
  21. #include <linux/errno.h>
  22. #include <linux/percpu.h>
  23. #include <linux/gfp.h>
  24. #include <linux/mm.h>
  25. #include <linux/miscdevice.h>
  26. #include <linux/vmalloc.h>
  27. #include <linux/reboot.h>
  28. #include <linux/debugfs.h>
  29. #include <linux/highmem.h>
  30. #include <linux/file.h>
  31. #include <linux/sysdev.h>
  32. #include <linux/cpu.h>
  33. #include <linux/sched.h>
  34. #include <linux/cpumask.h>
  35. #include <linux/smp.h>
  36. #include <linux/anon_inodes.h>
  37. #include <linux/profile.h>
  38. #include <linux/kvm_para.h>
  39. #include <linux/pagemap.h>
  40. #include <linux/mman.h>
  41. #include <asm/processor.h>
  42. #include <asm/io.h>
  43. #include <asm/uaccess.h>
  44. #include <asm/pgtable.h>
  45. MODULE_AUTHOR("Qumranet");
  46. MODULE_LICENSE("GPL");
  47. DEFINE_SPINLOCK(kvm_lock);
  48. LIST_HEAD(vm_list);
  49. static cpumask_t cpus_hardware_enabled;
  50. struct kmem_cache *kvm_vcpu_cache;
  51. EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  52. static __read_mostly struct preempt_ops kvm_preempt_ops;
  53. static struct dentry *debugfs_dir;
  54. static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  55. unsigned long arg);
  56. static inline int valid_vcpu(int n)
  57. {
  58. return likely(n >= 0 && n < KVM_MAX_VCPUS);
  59. }
  60. /*
  61. * Switches to specified vcpu, until a matching vcpu_put()
  62. */
  63. void vcpu_load(struct kvm_vcpu *vcpu)
  64. {
  65. int cpu;
  66. mutex_lock(&vcpu->mutex);
  67. cpu = get_cpu();
  68. preempt_notifier_register(&vcpu->preempt_notifier);
  69. kvm_arch_vcpu_load(vcpu, cpu);
  70. put_cpu();
  71. }
  72. void vcpu_put(struct kvm_vcpu *vcpu)
  73. {
  74. preempt_disable();
  75. kvm_arch_vcpu_put(vcpu);
  76. preempt_notifier_unregister(&vcpu->preempt_notifier);
  77. preempt_enable();
  78. mutex_unlock(&vcpu->mutex);
  79. }
  80. static void ack_flush(void *_completed)
  81. {
  82. }
  83. void kvm_flush_remote_tlbs(struct kvm *kvm)
  84. {
  85. int i, cpu;
  86. cpumask_t cpus;
  87. struct kvm_vcpu *vcpu;
  88. cpus_clear(cpus);
  89. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  90. vcpu = kvm->vcpus[i];
  91. if (!vcpu)
  92. continue;
  93. if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
  94. continue;
  95. cpu = vcpu->cpu;
  96. if (cpu != -1 && cpu != raw_smp_processor_id())
  97. cpu_set(cpu, cpus);
  98. }
  99. if (cpus_empty(cpus))
  100. return;
  101. ++kvm->stat.remote_tlb_flush;
  102. smp_call_function_mask(cpus, ack_flush, NULL, 1);
  103. }
  104. int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
  105. {
  106. struct page *page;
  107. int r;
  108. mutex_init(&vcpu->mutex);
  109. vcpu->cpu = -1;
  110. vcpu->kvm = kvm;
  111. vcpu->vcpu_id = id;
  112. init_waitqueue_head(&vcpu->wq);
  113. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  114. if (!page) {
  115. r = -ENOMEM;
  116. goto fail;
  117. }
  118. vcpu->run = page_address(page);
  119. r = kvm_arch_vcpu_init(vcpu);
  120. if (r < 0)
  121. goto fail_free_run;
  122. return 0;
  123. fail_free_run:
  124. free_page((unsigned long)vcpu->run);
  125. fail:
  126. return r;
  127. }
  128. EXPORT_SYMBOL_GPL(kvm_vcpu_init);
  129. void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
  130. {
  131. kvm_arch_vcpu_uninit(vcpu);
  132. free_page((unsigned long)vcpu->run);
  133. }
  134. EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
  135. static struct kvm *kvm_create_vm(void)
  136. {
  137. struct kvm *kvm = kvm_arch_create_vm();
  138. if (IS_ERR(kvm))
  139. goto out;
  140. kvm->mm = current->mm;
  141. atomic_inc(&kvm->mm->mm_count);
  142. spin_lock_init(&kvm->mmu_lock);
  143. kvm_io_bus_init(&kvm->pio_bus);
  144. mutex_init(&kvm->lock);
  145. kvm_io_bus_init(&kvm->mmio_bus);
  146. spin_lock(&kvm_lock);
  147. list_add(&kvm->vm_list, &vm_list);
  148. spin_unlock(&kvm_lock);
  149. out:
  150. return kvm;
  151. }
  152. /*
  153. * Free any memory in @free but not in @dont.
  154. */
  155. static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
  156. struct kvm_memory_slot *dont)
  157. {
  158. if (!dont || free->rmap != dont->rmap)
  159. vfree(free->rmap);
  160. if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
  161. vfree(free->dirty_bitmap);
  162. free->npages = 0;
  163. free->dirty_bitmap = NULL;
  164. free->rmap = NULL;
  165. }
  166. void kvm_free_physmem(struct kvm *kvm)
  167. {
  168. int i;
  169. for (i = 0; i < kvm->nmemslots; ++i)
  170. kvm_free_physmem_slot(&kvm->memslots[i], NULL);
  171. }
  172. static void kvm_destroy_vm(struct kvm *kvm)
  173. {
  174. struct mm_struct *mm = kvm->mm;
  175. spin_lock(&kvm_lock);
  176. list_del(&kvm->vm_list);
  177. spin_unlock(&kvm_lock);
  178. kvm_io_bus_destroy(&kvm->pio_bus);
  179. kvm_io_bus_destroy(&kvm->mmio_bus);
  180. kvm_arch_destroy_vm(kvm);
  181. mmdrop(mm);
  182. }
  183. static int kvm_vm_release(struct inode *inode, struct file *filp)
  184. {
  185. struct kvm *kvm = filp->private_data;
  186. kvm_destroy_vm(kvm);
  187. return 0;
  188. }
  189. /*
  190. * Allocate some memory and give it an address in the guest physical address
  191. * space.
  192. *
  193. * Discontiguous memory is allowed, mostly for framebuffers.
  194. *
  195. * Must be called holding mmap_sem for write.
  196. */
  197. int __kvm_set_memory_region(struct kvm *kvm,
  198. struct kvm_userspace_memory_region *mem,
  199. int user_alloc)
  200. {
  201. int r;
  202. gfn_t base_gfn;
  203. unsigned long npages;
  204. unsigned long i;
  205. struct kvm_memory_slot *memslot;
  206. struct kvm_memory_slot old, new;
  207. r = -EINVAL;
  208. /* General sanity checks */
  209. if (mem->memory_size & (PAGE_SIZE - 1))
  210. goto out;
  211. if (mem->guest_phys_addr & (PAGE_SIZE - 1))
  212. goto out;
  213. if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
  214. goto out;
  215. if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
  216. goto out;
  217. memslot = &kvm->memslots[mem->slot];
  218. base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
  219. npages = mem->memory_size >> PAGE_SHIFT;
  220. if (!npages)
  221. mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
  222. new = old = *memslot;
  223. new.base_gfn = base_gfn;
  224. new.npages = npages;
  225. new.flags = mem->flags;
  226. /* Disallow changing a memory slot's size. */
  227. r = -EINVAL;
  228. if (npages && old.npages && npages != old.npages)
  229. goto out_free;
  230. /* Check for overlaps */
  231. r = -EEXIST;
  232. for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
  233. struct kvm_memory_slot *s = &kvm->memslots[i];
  234. if (s == memslot)
  235. continue;
  236. if (!((base_gfn + npages <= s->base_gfn) ||
  237. (base_gfn >= s->base_gfn + s->npages)))
  238. goto out_free;
  239. }
  240. /* Free page dirty bitmap if unneeded */
  241. if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
  242. new.dirty_bitmap = NULL;
  243. r = -ENOMEM;
  244. /* Allocate if a slot is being created */
  245. if (npages && !new.rmap) {
  246. new.rmap = vmalloc(npages * sizeof(struct page *));
  247. if (!new.rmap)
  248. goto out_free;
  249. memset(new.rmap, 0, npages * sizeof(*new.rmap));
  250. new.user_alloc = user_alloc;
  251. new.userspace_addr = mem->userspace_addr;
  252. }
  253. /* Allocate page dirty bitmap if needed */
  254. if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
  255. unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
  256. new.dirty_bitmap = vmalloc(dirty_bytes);
  257. if (!new.dirty_bitmap)
  258. goto out_free;
  259. memset(new.dirty_bitmap, 0, dirty_bytes);
  260. }
  261. if (mem->slot >= kvm->nmemslots)
  262. kvm->nmemslots = mem->slot + 1;
  263. *memslot = new;
  264. r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
  265. if (r) {
  266. *memslot = old;
  267. goto out_free;
  268. }
  269. kvm_free_physmem_slot(&old, &new);
  270. return 0;
  271. out_free:
  272. kvm_free_physmem_slot(&new, &old);
  273. out:
  274. return r;
  275. }
  276. EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
  277. int kvm_set_memory_region(struct kvm *kvm,
  278. struct kvm_userspace_memory_region *mem,
  279. int user_alloc)
  280. {
  281. int r;
  282. down_write(&current->mm->mmap_sem);
  283. r = __kvm_set_memory_region(kvm, mem, user_alloc);
  284. up_write(&current->mm->mmap_sem);
  285. return r;
  286. }
  287. EXPORT_SYMBOL_GPL(kvm_set_memory_region);
  288. int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  289. struct
  290. kvm_userspace_memory_region *mem,
  291. int user_alloc)
  292. {
  293. if (mem->slot >= KVM_MEMORY_SLOTS)
  294. return -EINVAL;
  295. return kvm_set_memory_region(kvm, mem, user_alloc);
  296. }
  297. int kvm_get_dirty_log(struct kvm *kvm,
  298. struct kvm_dirty_log *log, int *is_dirty)
  299. {
  300. struct kvm_memory_slot *memslot;
  301. int r, i;
  302. int n;
  303. unsigned long any = 0;
  304. r = -EINVAL;
  305. if (log->slot >= KVM_MEMORY_SLOTS)
  306. goto out;
  307. memslot = &kvm->memslots[log->slot];
  308. r = -ENOENT;
  309. if (!memslot->dirty_bitmap)
  310. goto out;
  311. n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
  312. for (i = 0; !any && i < n/sizeof(long); ++i)
  313. any = memslot->dirty_bitmap[i];
  314. r = -EFAULT;
  315. if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
  316. goto out;
  317. if (any)
  318. *is_dirty = 1;
  319. r = 0;
  320. out:
  321. return r;
  322. }
  323. int is_error_page(struct page *page)
  324. {
  325. return page == bad_page;
  326. }
  327. EXPORT_SYMBOL_GPL(is_error_page);
  328. static inline unsigned long bad_hva(void)
  329. {
  330. return PAGE_OFFSET;
  331. }
  332. int kvm_is_error_hva(unsigned long addr)
  333. {
  334. return addr == bad_hva();
  335. }
  336. EXPORT_SYMBOL_GPL(kvm_is_error_hva);
  337. static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  338. {
  339. int i;
  340. for (i = 0; i < kvm->nmemslots; ++i) {
  341. struct kvm_memory_slot *memslot = &kvm->memslots[i];
  342. if (gfn >= memslot->base_gfn
  343. && gfn < memslot->base_gfn + memslot->npages)
  344. return memslot;
  345. }
  346. return NULL;
  347. }
  348. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  349. {
  350. gfn = unalias_gfn(kvm, gfn);
  351. return __gfn_to_memslot(kvm, gfn);
  352. }
  353. int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
  354. {
  355. int i;
  356. gfn = unalias_gfn(kvm, gfn);
  357. for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
  358. struct kvm_memory_slot *memslot = &kvm->memslots[i];
  359. if (gfn >= memslot->base_gfn
  360. && gfn < memslot->base_gfn + memslot->npages)
  361. return 1;
  362. }
  363. return 0;
  364. }
  365. EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
  366. static unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
  367. {
  368. struct kvm_memory_slot *slot;
  369. gfn = unalias_gfn(kvm, gfn);
  370. slot = __gfn_to_memslot(kvm, gfn);
  371. if (!slot)
  372. return bad_hva();
  373. return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
  374. }
  375. /*
  376. * Requires current->mm->mmap_sem to be held
  377. */
  378. struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
  379. {
  380. struct page *page[1];
  381. unsigned long addr;
  382. int npages;
  383. might_sleep();
  384. addr = gfn_to_hva(kvm, gfn);
  385. if (kvm_is_error_hva(addr)) {
  386. get_page(bad_page);
  387. return bad_page;
  388. }
  389. npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page,
  390. NULL);
  391. if (npages != 1) {
  392. get_page(bad_page);
  393. return bad_page;
  394. }
  395. return page[0];
  396. }
  397. EXPORT_SYMBOL_GPL(gfn_to_page);
  398. void kvm_release_page_clean(struct page *page)
  399. {
  400. put_page(page);
  401. }
  402. EXPORT_SYMBOL_GPL(kvm_release_page_clean);
  403. void kvm_release_page_dirty(struct page *page)
  404. {
  405. if (!PageReserved(page))
  406. SetPageDirty(page);
  407. put_page(page);
  408. }
  409. EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
  410. static int next_segment(unsigned long len, int offset)
  411. {
  412. if (len > PAGE_SIZE - offset)
  413. return PAGE_SIZE - offset;
  414. else
  415. return len;
  416. }
  417. int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
  418. int len)
  419. {
  420. int r;
  421. unsigned long addr;
  422. addr = gfn_to_hva(kvm, gfn);
  423. if (kvm_is_error_hva(addr))
  424. return -EFAULT;
  425. r = copy_from_user(data, (void __user *)addr + offset, len);
  426. if (r)
  427. return -EFAULT;
  428. return 0;
  429. }
  430. EXPORT_SYMBOL_GPL(kvm_read_guest_page);
  431. int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
  432. {
  433. gfn_t gfn = gpa >> PAGE_SHIFT;
  434. int seg;
  435. int offset = offset_in_page(gpa);
  436. int ret;
  437. while ((seg = next_segment(len, offset)) != 0) {
  438. ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
  439. if (ret < 0)
  440. return ret;
  441. offset = 0;
  442. len -= seg;
  443. data += seg;
  444. ++gfn;
  445. }
  446. return 0;
  447. }
  448. EXPORT_SYMBOL_GPL(kvm_read_guest);
  449. int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
  450. unsigned long len)
  451. {
  452. int r;
  453. unsigned long addr;
  454. gfn_t gfn = gpa >> PAGE_SHIFT;
  455. int offset = offset_in_page(gpa);
  456. addr = gfn_to_hva(kvm, gfn);
  457. if (kvm_is_error_hva(addr))
  458. return -EFAULT;
  459. r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
  460. if (r)
  461. return -EFAULT;
  462. return 0;
  463. }
  464. EXPORT_SYMBOL(kvm_read_guest_atomic);
  465. int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
  466. int offset, int len)
  467. {
  468. int r;
  469. unsigned long addr;
  470. addr = gfn_to_hva(kvm, gfn);
  471. if (kvm_is_error_hva(addr))
  472. return -EFAULT;
  473. r = copy_to_user((void __user *)addr + offset, data, len);
  474. if (r)
  475. return -EFAULT;
  476. mark_page_dirty(kvm, gfn);
  477. return 0;
  478. }
  479. EXPORT_SYMBOL_GPL(kvm_write_guest_page);
  480. int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
  481. unsigned long len)
  482. {
  483. gfn_t gfn = gpa >> PAGE_SHIFT;
  484. int seg;
  485. int offset = offset_in_page(gpa);
  486. int ret;
  487. while ((seg = next_segment(len, offset)) != 0) {
  488. ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
  489. if (ret < 0)
  490. return ret;
  491. offset = 0;
  492. len -= seg;
  493. data += seg;
  494. ++gfn;
  495. }
  496. return 0;
  497. }
  498. int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
  499. {
  500. return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
  501. }
  502. EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
  503. int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
  504. {
  505. gfn_t gfn = gpa >> PAGE_SHIFT;
  506. int seg;
  507. int offset = offset_in_page(gpa);
  508. int ret;
  509. while ((seg = next_segment(len, offset)) != 0) {
  510. ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
  511. if (ret < 0)
  512. return ret;
  513. offset = 0;
  514. len -= seg;
  515. ++gfn;
  516. }
  517. return 0;
  518. }
  519. EXPORT_SYMBOL_GPL(kvm_clear_guest);
  520. void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
  521. {
  522. struct kvm_memory_slot *memslot;
  523. gfn = unalias_gfn(kvm, gfn);
  524. memslot = __gfn_to_memslot(kvm, gfn);
  525. if (memslot && memslot->dirty_bitmap) {
  526. unsigned long rel_gfn = gfn - memslot->base_gfn;
  527. /* avoid RMW */
  528. if (!test_bit(rel_gfn, memslot->dirty_bitmap))
  529. set_bit(rel_gfn, memslot->dirty_bitmap);
  530. }
  531. }
  532. /*
  533. * The vCPU has executed a HLT instruction with in-kernel mode enabled.
  534. */
  535. void kvm_vcpu_block(struct kvm_vcpu *vcpu)
  536. {
  537. DECLARE_WAITQUEUE(wait, current);
  538. add_wait_queue(&vcpu->wq, &wait);
  539. /*
  540. * We will block until either an interrupt or a signal wakes us up
  541. */
  542. while (!kvm_cpu_has_interrupt(vcpu)
  543. && !signal_pending(current)
  544. && !kvm_arch_vcpu_runnable(vcpu)) {
  545. set_current_state(TASK_INTERRUPTIBLE);
  546. vcpu_put(vcpu);
  547. schedule();
  548. vcpu_load(vcpu);
  549. }
  550. __set_current_state(TASK_RUNNING);
  551. remove_wait_queue(&vcpu->wq, &wait);
  552. }
  553. void kvm_resched(struct kvm_vcpu *vcpu)
  554. {
  555. if (!need_resched())
  556. return;
  557. cond_resched();
  558. }
  559. EXPORT_SYMBOL_GPL(kvm_resched);
  560. static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  561. {
  562. struct kvm_vcpu *vcpu = vma->vm_file->private_data;
  563. struct page *page;
  564. if (vmf->pgoff == 0)
  565. page = virt_to_page(vcpu->run);
  566. else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
  567. page = virt_to_page(vcpu->arch.pio_data);
  568. else
  569. return VM_FAULT_SIGBUS;
  570. get_page(page);
  571. vmf->page = page;
  572. return 0;
  573. }
  574. static struct vm_operations_struct kvm_vcpu_vm_ops = {
  575. .fault = kvm_vcpu_fault,
  576. };
  577. static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
  578. {
  579. vma->vm_ops = &kvm_vcpu_vm_ops;
  580. return 0;
  581. }
  582. static int kvm_vcpu_release(struct inode *inode, struct file *filp)
  583. {
  584. struct kvm_vcpu *vcpu = filp->private_data;
  585. fput(vcpu->kvm->filp);
  586. return 0;
  587. }
  588. static struct file_operations kvm_vcpu_fops = {
  589. .release = kvm_vcpu_release,
  590. .unlocked_ioctl = kvm_vcpu_ioctl,
  591. .compat_ioctl = kvm_vcpu_ioctl,
  592. .mmap = kvm_vcpu_mmap,
  593. };
  594. /*
  595. * Allocates an inode for the vcpu.
  596. */
  597. static int create_vcpu_fd(struct kvm_vcpu *vcpu)
  598. {
  599. int fd, r;
  600. struct inode *inode;
  601. struct file *file;
  602. r = anon_inode_getfd(&fd, &inode, &file,
  603. "kvm-vcpu", &kvm_vcpu_fops, vcpu);
  604. if (r)
  605. return r;
  606. atomic_inc(&vcpu->kvm->filp->f_count);
  607. return fd;
  608. }
  609. /*
  610. * Creates some virtual cpus. Good luck creating more than one.
  611. */
  612. static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
  613. {
  614. int r;
  615. struct kvm_vcpu *vcpu;
  616. if (!valid_vcpu(n))
  617. return -EINVAL;
  618. vcpu = kvm_arch_vcpu_create(kvm, n);
  619. if (IS_ERR(vcpu))
  620. return PTR_ERR(vcpu);
  621. preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
  622. r = kvm_arch_vcpu_setup(vcpu);
  623. if (r)
  624. goto vcpu_destroy;
  625. mutex_lock(&kvm->lock);
  626. if (kvm->vcpus[n]) {
  627. r = -EEXIST;
  628. mutex_unlock(&kvm->lock);
  629. goto vcpu_destroy;
  630. }
  631. kvm->vcpus[n] = vcpu;
  632. mutex_unlock(&kvm->lock);
  633. /* Now it's all set up, let userspace reach it */
  634. r = create_vcpu_fd(vcpu);
  635. if (r < 0)
  636. goto unlink;
  637. return r;
  638. unlink:
  639. mutex_lock(&kvm->lock);
  640. kvm->vcpus[n] = NULL;
  641. mutex_unlock(&kvm->lock);
  642. vcpu_destroy:
  643. kvm_arch_vcpu_destroy(vcpu);
  644. return r;
  645. }
  646. static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
  647. {
  648. if (sigset) {
  649. sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
  650. vcpu->sigset_active = 1;
  651. vcpu->sigset = *sigset;
  652. } else
  653. vcpu->sigset_active = 0;
  654. return 0;
  655. }
  656. static long kvm_vcpu_ioctl(struct file *filp,
  657. unsigned int ioctl, unsigned long arg)
  658. {
  659. struct kvm_vcpu *vcpu = filp->private_data;
  660. void __user *argp = (void __user *)arg;
  661. int r;
  662. if (vcpu->kvm->mm != current->mm)
  663. return -EIO;
  664. switch (ioctl) {
  665. case KVM_RUN:
  666. r = -EINVAL;
  667. if (arg)
  668. goto out;
  669. r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
  670. break;
  671. case KVM_GET_REGS: {
  672. struct kvm_regs kvm_regs;
  673. memset(&kvm_regs, 0, sizeof kvm_regs);
  674. r = kvm_arch_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
  675. if (r)
  676. goto out;
  677. r = -EFAULT;
  678. if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
  679. goto out;
  680. r = 0;
  681. break;
  682. }
  683. case KVM_SET_REGS: {
  684. struct kvm_regs kvm_regs;
  685. r = -EFAULT;
  686. if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
  687. goto out;
  688. r = kvm_arch_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
  689. if (r)
  690. goto out;
  691. r = 0;
  692. break;
  693. }
  694. case KVM_GET_SREGS: {
  695. struct kvm_sregs kvm_sregs;
  696. memset(&kvm_sregs, 0, sizeof kvm_sregs);
  697. r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
  698. if (r)
  699. goto out;
  700. r = -EFAULT;
  701. if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
  702. goto out;
  703. r = 0;
  704. break;
  705. }
  706. case KVM_SET_SREGS: {
  707. struct kvm_sregs kvm_sregs;
  708. r = -EFAULT;
  709. if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
  710. goto out;
  711. r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
  712. if (r)
  713. goto out;
  714. r = 0;
  715. break;
  716. }
  717. case KVM_TRANSLATE: {
  718. struct kvm_translation tr;
  719. r = -EFAULT;
  720. if (copy_from_user(&tr, argp, sizeof tr))
  721. goto out;
  722. r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
  723. if (r)
  724. goto out;
  725. r = -EFAULT;
  726. if (copy_to_user(argp, &tr, sizeof tr))
  727. goto out;
  728. r = 0;
  729. break;
  730. }
  731. case KVM_DEBUG_GUEST: {
  732. struct kvm_debug_guest dbg;
  733. r = -EFAULT;
  734. if (copy_from_user(&dbg, argp, sizeof dbg))
  735. goto out;
  736. r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
  737. if (r)
  738. goto out;
  739. r = 0;
  740. break;
  741. }
  742. case KVM_SET_SIGNAL_MASK: {
  743. struct kvm_signal_mask __user *sigmask_arg = argp;
  744. struct kvm_signal_mask kvm_sigmask;
  745. sigset_t sigset, *p;
  746. p = NULL;
  747. if (argp) {
  748. r = -EFAULT;
  749. if (copy_from_user(&kvm_sigmask, argp,
  750. sizeof kvm_sigmask))
  751. goto out;
  752. r = -EINVAL;
  753. if (kvm_sigmask.len != sizeof sigset)
  754. goto out;
  755. r = -EFAULT;
  756. if (copy_from_user(&sigset, sigmask_arg->sigset,
  757. sizeof sigset))
  758. goto out;
  759. p = &sigset;
  760. }
  761. r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
  762. break;
  763. }
  764. case KVM_GET_FPU: {
  765. struct kvm_fpu fpu;
  766. memset(&fpu, 0, sizeof fpu);
  767. r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu);
  768. if (r)
  769. goto out;
  770. r = -EFAULT;
  771. if (copy_to_user(argp, &fpu, sizeof fpu))
  772. goto out;
  773. r = 0;
  774. break;
  775. }
  776. case KVM_SET_FPU: {
  777. struct kvm_fpu fpu;
  778. r = -EFAULT;
  779. if (copy_from_user(&fpu, argp, sizeof fpu))
  780. goto out;
  781. r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu);
  782. if (r)
  783. goto out;
  784. r = 0;
  785. break;
  786. }
  787. default:
  788. r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  789. }
  790. out:
  791. return r;
  792. }
  793. static long kvm_vm_ioctl(struct file *filp,
  794. unsigned int ioctl, unsigned long arg)
  795. {
  796. struct kvm *kvm = filp->private_data;
  797. void __user *argp = (void __user *)arg;
  798. int r;
  799. if (kvm->mm != current->mm)
  800. return -EIO;
  801. switch (ioctl) {
  802. case KVM_CREATE_VCPU:
  803. r = kvm_vm_ioctl_create_vcpu(kvm, arg);
  804. if (r < 0)
  805. goto out;
  806. break;
  807. case KVM_SET_USER_MEMORY_REGION: {
  808. struct kvm_userspace_memory_region kvm_userspace_mem;
  809. r = -EFAULT;
  810. if (copy_from_user(&kvm_userspace_mem, argp,
  811. sizeof kvm_userspace_mem))
  812. goto out;
  813. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
  814. if (r)
  815. goto out;
  816. break;
  817. }
  818. case KVM_GET_DIRTY_LOG: {
  819. struct kvm_dirty_log log;
  820. r = -EFAULT;
  821. if (copy_from_user(&log, argp, sizeof log))
  822. goto out;
  823. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  824. if (r)
  825. goto out;
  826. break;
  827. }
  828. default:
  829. r = kvm_arch_vm_ioctl(filp, ioctl, arg);
  830. }
  831. out:
  832. return r;
  833. }
  834. static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  835. {
  836. struct kvm *kvm = vma->vm_file->private_data;
  837. struct page *page;
  838. if (!kvm_is_visible_gfn(kvm, vmf->pgoff))
  839. return VM_FAULT_SIGBUS;
  840. page = gfn_to_page(kvm, vmf->pgoff);
  841. if (is_error_page(page)) {
  842. kvm_release_page_clean(page);
  843. return VM_FAULT_SIGBUS;
  844. }
  845. vmf->page = page;
  846. return 0;
  847. }
  848. static struct vm_operations_struct kvm_vm_vm_ops = {
  849. .fault = kvm_vm_fault,
  850. };
  851. static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
  852. {
  853. vma->vm_ops = &kvm_vm_vm_ops;
  854. return 0;
  855. }
  856. static struct file_operations kvm_vm_fops = {
  857. .release = kvm_vm_release,
  858. .unlocked_ioctl = kvm_vm_ioctl,
  859. .compat_ioctl = kvm_vm_ioctl,
  860. .mmap = kvm_vm_mmap,
  861. };
  862. static int kvm_dev_ioctl_create_vm(void)
  863. {
  864. int fd, r;
  865. struct inode *inode;
  866. struct file *file;
  867. struct kvm *kvm;
  868. kvm = kvm_create_vm();
  869. if (IS_ERR(kvm))
  870. return PTR_ERR(kvm);
  871. r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
  872. if (r) {
  873. kvm_destroy_vm(kvm);
  874. return r;
  875. }
  876. kvm->filp = file;
  877. return fd;
  878. }
  879. static long kvm_dev_ioctl(struct file *filp,
  880. unsigned int ioctl, unsigned long arg)
  881. {
  882. void __user *argp = (void __user *)arg;
  883. long r = -EINVAL;
  884. switch (ioctl) {
  885. case KVM_GET_API_VERSION:
  886. r = -EINVAL;
  887. if (arg)
  888. goto out;
  889. r = KVM_API_VERSION;
  890. break;
  891. case KVM_CREATE_VM:
  892. r = -EINVAL;
  893. if (arg)
  894. goto out;
  895. r = kvm_dev_ioctl_create_vm();
  896. break;
  897. case KVM_CHECK_EXTENSION:
  898. r = kvm_dev_ioctl_check_extension((long)argp);
  899. break;
  900. case KVM_GET_VCPU_MMAP_SIZE:
  901. r = -EINVAL;
  902. if (arg)
  903. goto out;
  904. r = 2 * PAGE_SIZE;
  905. break;
  906. default:
  907. return kvm_arch_dev_ioctl(filp, ioctl, arg);
  908. }
  909. out:
  910. return r;
  911. }
  912. static struct file_operations kvm_chardev_ops = {
  913. .unlocked_ioctl = kvm_dev_ioctl,
  914. .compat_ioctl = kvm_dev_ioctl,
  915. };
  916. static struct miscdevice kvm_dev = {
  917. KVM_MINOR,
  918. "kvm",
  919. &kvm_chardev_ops,
  920. };
  921. static void hardware_enable(void *junk)
  922. {
  923. int cpu = raw_smp_processor_id();
  924. if (cpu_isset(cpu, cpus_hardware_enabled))
  925. return;
  926. cpu_set(cpu, cpus_hardware_enabled);
  927. kvm_arch_hardware_enable(NULL);
  928. }
  929. static void hardware_disable(void *junk)
  930. {
  931. int cpu = raw_smp_processor_id();
  932. if (!cpu_isset(cpu, cpus_hardware_enabled))
  933. return;
  934. cpu_clear(cpu, cpus_hardware_enabled);
  935. decache_vcpus_on_cpu(cpu);
  936. kvm_arch_hardware_disable(NULL);
  937. }
  938. static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  939. void *v)
  940. {
  941. int cpu = (long)v;
  942. val &= ~CPU_TASKS_FROZEN;
  943. switch (val) {
  944. case CPU_DYING:
  945. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  946. cpu);
  947. hardware_disable(NULL);
  948. break;
  949. case CPU_UP_CANCELED:
  950. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  951. cpu);
  952. smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
  953. break;
  954. case CPU_ONLINE:
  955. printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
  956. cpu);
  957. smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
  958. break;
  959. }
  960. return NOTIFY_OK;
  961. }
  962. static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
  963. void *v)
  964. {
  965. if (val == SYS_RESTART) {
  966. /*
  967. * Some (well, at least mine) BIOSes hang on reboot if
  968. * in vmx root mode.
  969. */
  970. printk(KERN_INFO "kvm: exiting hardware virtualization\n");
  971. on_each_cpu(hardware_disable, NULL, 0, 1);
  972. }
  973. return NOTIFY_OK;
  974. }
  975. static struct notifier_block kvm_reboot_notifier = {
  976. .notifier_call = kvm_reboot,
  977. .priority = 0,
  978. };
  979. void kvm_io_bus_init(struct kvm_io_bus *bus)
  980. {
  981. memset(bus, 0, sizeof(*bus));
  982. }
  983. void kvm_io_bus_destroy(struct kvm_io_bus *bus)
  984. {
  985. int i;
  986. for (i = 0; i < bus->dev_count; i++) {
  987. struct kvm_io_device *pos = bus->devs[i];
  988. kvm_iodevice_destructor(pos);
  989. }
  990. }
  991. struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
  992. {
  993. int i;
  994. for (i = 0; i < bus->dev_count; i++) {
  995. struct kvm_io_device *pos = bus->devs[i];
  996. if (pos->in_range(pos, addr))
  997. return pos;
  998. }
  999. return NULL;
  1000. }
  1001. void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
  1002. {
  1003. BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
  1004. bus->devs[bus->dev_count++] = dev;
  1005. }
  1006. static struct notifier_block kvm_cpu_notifier = {
  1007. .notifier_call = kvm_cpu_hotplug,
  1008. .priority = 20, /* must be > scheduler priority */
  1009. };
  1010. static int vm_stat_get(void *_offset, u64 *val)
  1011. {
  1012. unsigned offset = (long)_offset;
  1013. struct kvm *kvm;
  1014. *val = 0;
  1015. spin_lock(&kvm_lock);
  1016. list_for_each_entry(kvm, &vm_list, vm_list)
  1017. *val += *(u32 *)((void *)kvm + offset);
  1018. spin_unlock(&kvm_lock);
  1019. return 0;
  1020. }
  1021. DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
  1022. static int vcpu_stat_get(void *_offset, u64 *val)
  1023. {
  1024. unsigned offset = (long)_offset;
  1025. struct kvm *kvm;
  1026. struct kvm_vcpu *vcpu;
  1027. int i;
  1028. *val = 0;
  1029. spin_lock(&kvm_lock);
  1030. list_for_each_entry(kvm, &vm_list, vm_list)
  1031. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  1032. vcpu = kvm->vcpus[i];
  1033. if (vcpu)
  1034. *val += *(u32 *)((void *)vcpu + offset);
  1035. }
  1036. spin_unlock(&kvm_lock);
  1037. return 0;
  1038. }
  1039. DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
  1040. static struct file_operations *stat_fops[] = {
  1041. [KVM_STAT_VCPU] = &vcpu_stat_fops,
  1042. [KVM_STAT_VM] = &vm_stat_fops,
  1043. };
  1044. static void kvm_init_debug(void)
  1045. {
  1046. struct kvm_stats_debugfs_item *p;
  1047. debugfs_dir = debugfs_create_dir("kvm", NULL);
  1048. for (p = debugfs_entries; p->name; ++p)
  1049. p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
  1050. (void *)(long)p->offset,
  1051. stat_fops[p->kind]);
  1052. }
  1053. static void kvm_exit_debug(void)
  1054. {
  1055. struct kvm_stats_debugfs_item *p;
  1056. for (p = debugfs_entries; p->name; ++p)
  1057. debugfs_remove(p->dentry);
  1058. debugfs_remove(debugfs_dir);
  1059. }
  1060. static int kvm_suspend(struct sys_device *dev, pm_message_t state)
  1061. {
  1062. hardware_disable(NULL);
  1063. return 0;
  1064. }
  1065. static int kvm_resume(struct sys_device *dev)
  1066. {
  1067. hardware_enable(NULL);
  1068. return 0;
  1069. }
  1070. static struct sysdev_class kvm_sysdev_class = {
  1071. .name = "kvm",
  1072. .suspend = kvm_suspend,
  1073. .resume = kvm_resume,
  1074. };
  1075. static struct sys_device kvm_sysdev = {
  1076. .id = 0,
  1077. .cls = &kvm_sysdev_class,
  1078. };
  1079. struct page *bad_page;
  1080. static inline
  1081. struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
  1082. {
  1083. return container_of(pn, struct kvm_vcpu, preempt_notifier);
  1084. }
  1085. static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
  1086. {
  1087. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  1088. kvm_arch_vcpu_load(vcpu, cpu);
  1089. }
  1090. static void kvm_sched_out(struct preempt_notifier *pn,
  1091. struct task_struct *next)
  1092. {
  1093. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  1094. kvm_arch_vcpu_put(vcpu);
  1095. }
  1096. int kvm_init(void *opaque, unsigned int vcpu_size,
  1097. struct module *module)
  1098. {
  1099. int r;
  1100. int cpu;
  1101. kvm_init_debug();
  1102. r = kvm_arch_init(opaque);
  1103. if (r)
  1104. goto out_fail;
  1105. bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  1106. if (bad_page == NULL) {
  1107. r = -ENOMEM;
  1108. goto out;
  1109. }
  1110. r = kvm_arch_hardware_setup();
  1111. if (r < 0)
  1112. goto out_free_0;
  1113. for_each_online_cpu(cpu) {
  1114. smp_call_function_single(cpu,
  1115. kvm_arch_check_processor_compat,
  1116. &r, 0, 1);
  1117. if (r < 0)
  1118. goto out_free_1;
  1119. }
  1120. on_each_cpu(hardware_enable, NULL, 0, 1);
  1121. r = register_cpu_notifier(&kvm_cpu_notifier);
  1122. if (r)
  1123. goto out_free_2;
  1124. register_reboot_notifier(&kvm_reboot_notifier);
  1125. r = sysdev_class_register(&kvm_sysdev_class);
  1126. if (r)
  1127. goto out_free_3;
  1128. r = sysdev_register(&kvm_sysdev);
  1129. if (r)
  1130. goto out_free_4;
  1131. /* A kmem cache lets us meet the alignment requirements of fx_save. */
  1132. kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
  1133. __alignof__(struct kvm_vcpu),
  1134. 0, NULL);
  1135. if (!kvm_vcpu_cache) {
  1136. r = -ENOMEM;
  1137. goto out_free_5;
  1138. }
  1139. kvm_chardev_ops.owner = module;
  1140. r = misc_register(&kvm_dev);
  1141. if (r) {
  1142. printk(KERN_ERR "kvm: misc device register failed\n");
  1143. goto out_free;
  1144. }
  1145. kvm_preempt_ops.sched_in = kvm_sched_in;
  1146. kvm_preempt_ops.sched_out = kvm_sched_out;
  1147. return 0;
  1148. out_free:
  1149. kmem_cache_destroy(kvm_vcpu_cache);
  1150. out_free_5:
  1151. sysdev_unregister(&kvm_sysdev);
  1152. out_free_4:
  1153. sysdev_class_unregister(&kvm_sysdev_class);
  1154. out_free_3:
  1155. unregister_reboot_notifier(&kvm_reboot_notifier);
  1156. unregister_cpu_notifier(&kvm_cpu_notifier);
  1157. out_free_2:
  1158. on_each_cpu(hardware_disable, NULL, 0, 1);
  1159. out_free_1:
  1160. kvm_arch_hardware_unsetup();
  1161. out_free_0:
  1162. __free_page(bad_page);
  1163. out:
  1164. kvm_arch_exit();
  1165. kvm_exit_debug();
  1166. out_fail:
  1167. return r;
  1168. }
  1169. EXPORT_SYMBOL_GPL(kvm_init);
  1170. void kvm_exit(void)
  1171. {
  1172. misc_deregister(&kvm_dev);
  1173. kmem_cache_destroy(kvm_vcpu_cache);
  1174. sysdev_unregister(&kvm_sysdev);
  1175. sysdev_class_unregister(&kvm_sysdev_class);
  1176. unregister_reboot_notifier(&kvm_reboot_notifier);
  1177. unregister_cpu_notifier(&kvm_cpu_notifier);
  1178. on_each_cpu(hardware_disable, NULL, 0, 1);
  1179. kvm_arch_hardware_unsetup();
  1180. kvm_arch_exit();
  1181. kvm_exit_debug();
  1182. __free_page(bad_page);
  1183. }
  1184. EXPORT_SYMBOL_GPL(kvm_exit);