kvm_main.c 32 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. *
  9. * Authors:
  10. * Avi Kivity <avi@qumranet.com>
  11. * Yaniv Kamay <yaniv@qumranet.com>
  12. *
  13. * This work is licensed under the terms of the GNU GPL, version 2. See
  14. * the COPYING file in the top-level directory.
  15. *
  16. */
  17. #include "iodev.h"
  18. #include <linux/kvm_host.h>
  19. #include <linux/kvm.h>
  20. #include <linux/module.h>
  21. #include <linux/errno.h>
  22. #include <linux/percpu.h>
  23. #include <linux/gfp.h>
  24. #include <linux/mm.h>
  25. #include <linux/miscdevice.h>
  26. #include <linux/vmalloc.h>
  27. #include <linux/reboot.h>
  28. #include <linux/debugfs.h>
  29. #include <linux/highmem.h>
  30. #include <linux/file.h>
  31. #include <linux/sysdev.h>
  32. #include <linux/cpu.h>
  33. #include <linux/sched.h>
  34. #include <linux/cpumask.h>
  35. #include <linux/smp.h>
  36. #include <linux/anon_inodes.h>
  37. #include <linux/profile.h>
  38. #include <linux/kvm_para.h>
  39. #include <linux/pagemap.h>
  40. #include <linux/mman.h>
  41. #include <linux/swap.h>
  42. #include <asm/processor.h>
  43. #include <asm/io.h>
  44. #include <asm/uaccess.h>
  45. #include <asm/pgtable.h>
  46. MODULE_AUTHOR("Qumranet");
  47. MODULE_LICENSE("GPL");
  48. DEFINE_SPINLOCK(kvm_lock);
  49. LIST_HEAD(vm_list);
  50. static cpumask_t cpus_hardware_enabled;
  51. struct kmem_cache *kvm_vcpu_cache;
  52. EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  53. static __read_mostly struct preempt_ops kvm_preempt_ops;
  54. struct dentry *kvm_debugfs_dir;
  55. static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  56. unsigned long arg);
  57. static inline int valid_vcpu(int n)
  58. {
  59. return likely(n >= 0 && n < KVM_MAX_VCPUS);
  60. }
  61. /*
  62. * Switches to specified vcpu, until a matching vcpu_put()
  63. */
  64. void vcpu_load(struct kvm_vcpu *vcpu)
  65. {
  66. int cpu;
  67. mutex_lock(&vcpu->mutex);
  68. cpu = get_cpu();
  69. preempt_notifier_register(&vcpu->preempt_notifier);
  70. kvm_arch_vcpu_load(vcpu, cpu);
  71. put_cpu();
  72. }
  73. void vcpu_put(struct kvm_vcpu *vcpu)
  74. {
  75. preempt_disable();
  76. kvm_arch_vcpu_put(vcpu);
  77. preempt_notifier_unregister(&vcpu->preempt_notifier);
  78. preempt_enable();
  79. mutex_unlock(&vcpu->mutex);
  80. }
  81. static void ack_flush(void *_completed)
  82. {
  83. }
  84. void kvm_flush_remote_tlbs(struct kvm *kvm)
  85. {
  86. int i, cpu;
  87. cpumask_t cpus;
  88. struct kvm_vcpu *vcpu;
  89. cpus_clear(cpus);
  90. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  91. vcpu = kvm->vcpus[i];
  92. if (!vcpu)
  93. continue;
  94. if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
  95. continue;
  96. cpu = vcpu->cpu;
  97. if (cpu != -1 && cpu != raw_smp_processor_id())
  98. cpu_set(cpu, cpus);
  99. }
  100. if (cpus_empty(cpus))
  101. return;
  102. ++kvm->stat.remote_tlb_flush;
  103. smp_call_function_mask(cpus, ack_flush, NULL, 1);
  104. }
  105. void kvm_reload_remote_mmus(struct kvm *kvm)
  106. {
  107. int i, cpu;
  108. cpumask_t cpus;
  109. struct kvm_vcpu *vcpu;
  110. cpus_clear(cpus);
  111. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  112. vcpu = kvm->vcpus[i];
  113. if (!vcpu)
  114. continue;
  115. if (test_and_set_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
  116. continue;
  117. cpu = vcpu->cpu;
  118. if (cpu != -1 && cpu != raw_smp_processor_id())
  119. cpu_set(cpu, cpus);
  120. }
  121. if (cpus_empty(cpus))
  122. return;
  123. smp_call_function_mask(cpus, ack_flush, NULL, 1);
  124. }
  125. int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
  126. {
  127. struct page *page;
  128. int r;
  129. mutex_init(&vcpu->mutex);
  130. vcpu->cpu = -1;
  131. vcpu->kvm = kvm;
  132. vcpu->vcpu_id = id;
  133. init_waitqueue_head(&vcpu->wq);
  134. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  135. if (!page) {
  136. r = -ENOMEM;
  137. goto fail;
  138. }
  139. vcpu->run = page_address(page);
  140. r = kvm_arch_vcpu_init(vcpu);
  141. if (r < 0)
  142. goto fail_free_run;
  143. return 0;
  144. fail_free_run:
  145. free_page((unsigned long)vcpu->run);
  146. fail:
  147. return r;
  148. }
  149. EXPORT_SYMBOL_GPL(kvm_vcpu_init);
  150. void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
  151. {
  152. kvm_arch_vcpu_uninit(vcpu);
  153. free_page((unsigned long)vcpu->run);
  154. }
  155. EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
  156. static struct kvm *kvm_create_vm(void)
  157. {
  158. struct kvm *kvm = kvm_arch_create_vm();
  159. if (IS_ERR(kvm))
  160. goto out;
  161. kvm->mm = current->mm;
  162. atomic_inc(&kvm->mm->mm_count);
  163. spin_lock_init(&kvm->mmu_lock);
  164. kvm_io_bus_init(&kvm->pio_bus);
  165. mutex_init(&kvm->lock);
  166. kvm_io_bus_init(&kvm->mmio_bus);
  167. init_rwsem(&kvm->slots_lock);
  168. atomic_set(&kvm->users_count, 1);
  169. spin_lock(&kvm_lock);
  170. list_add(&kvm->vm_list, &vm_list);
  171. spin_unlock(&kvm_lock);
  172. out:
  173. return kvm;
  174. }
  175. /*
  176. * Free any memory in @free but not in @dont.
  177. */
  178. static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
  179. struct kvm_memory_slot *dont)
  180. {
  181. if (!dont || free->rmap != dont->rmap)
  182. vfree(free->rmap);
  183. if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
  184. vfree(free->dirty_bitmap);
  185. if (!dont || free->lpage_info != dont->lpage_info)
  186. vfree(free->lpage_info);
  187. free->npages = 0;
  188. free->dirty_bitmap = NULL;
  189. free->rmap = NULL;
  190. free->lpage_info = NULL;
  191. }
  192. void kvm_free_physmem(struct kvm *kvm)
  193. {
  194. int i;
  195. for (i = 0; i < kvm->nmemslots; ++i)
  196. kvm_free_physmem_slot(&kvm->memslots[i], NULL);
  197. }
  198. static void kvm_destroy_vm(struct kvm *kvm)
  199. {
  200. struct mm_struct *mm = kvm->mm;
  201. spin_lock(&kvm_lock);
  202. list_del(&kvm->vm_list);
  203. spin_unlock(&kvm_lock);
  204. kvm_io_bus_destroy(&kvm->pio_bus);
  205. kvm_io_bus_destroy(&kvm->mmio_bus);
  206. kvm_arch_destroy_vm(kvm);
  207. mmdrop(mm);
  208. }
  209. void kvm_get_kvm(struct kvm *kvm)
  210. {
  211. atomic_inc(&kvm->users_count);
  212. }
  213. EXPORT_SYMBOL_GPL(kvm_get_kvm);
  214. void kvm_put_kvm(struct kvm *kvm)
  215. {
  216. if (atomic_dec_and_test(&kvm->users_count))
  217. kvm_destroy_vm(kvm);
  218. }
  219. EXPORT_SYMBOL_GPL(kvm_put_kvm);
  220. static int kvm_vm_release(struct inode *inode, struct file *filp)
  221. {
  222. struct kvm *kvm = filp->private_data;
  223. kvm_put_kvm(kvm);
  224. return 0;
  225. }
  226. /*
  227. * Allocate some memory and give it an address in the guest physical address
  228. * space.
  229. *
  230. * Discontiguous memory is allowed, mostly for framebuffers.
  231. *
  232. * Must be called holding mmap_sem for write.
  233. */
  234. int __kvm_set_memory_region(struct kvm *kvm,
  235. struct kvm_userspace_memory_region *mem,
  236. int user_alloc)
  237. {
  238. int r;
  239. gfn_t base_gfn;
  240. unsigned long npages;
  241. unsigned long i;
  242. struct kvm_memory_slot *memslot;
  243. struct kvm_memory_slot old, new;
  244. r = -EINVAL;
  245. /* General sanity checks */
  246. if (mem->memory_size & (PAGE_SIZE - 1))
  247. goto out;
  248. if (mem->guest_phys_addr & (PAGE_SIZE - 1))
  249. goto out;
  250. if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
  251. goto out;
  252. if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
  253. goto out;
  254. memslot = &kvm->memslots[mem->slot];
  255. base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
  256. npages = mem->memory_size >> PAGE_SHIFT;
  257. if (!npages)
  258. mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
  259. new = old = *memslot;
  260. new.base_gfn = base_gfn;
  261. new.npages = npages;
  262. new.flags = mem->flags;
  263. /* Disallow changing a memory slot's size. */
  264. r = -EINVAL;
  265. if (npages && old.npages && npages != old.npages)
  266. goto out_free;
  267. /* Check for overlaps */
  268. r = -EEXIST;
  269. for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
  270. struct kvm_memory_slot *s = &kvm->memslots[i];
  271. if (s == memslot)
  272. continue;
  273. if (!((base_gfn + npages <= s->base_gfn) ||
  274. (base_gfn >= s->base_gfn + s->npages)))
  275. goto out_free;
  276. }
  277. /* Free page dirty bitmap if unneeded */
  278. if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
  279. new.dirty_bitmap = NULL;
  280. r = -ENOMEM;
  281. /* Allocate if a slot is being created */
  282. if (npages && !new.rmap) {
  283. new.rmap = vmalloc(npages * sizeof(struct page *));
  284. if (!new.rmap)
  285. goto out_free;
  286. memset(new.rmap, 0, npages * sizeof(*new.rmap));
  287. new.user_alloc = user_alloc;
  288. new.userspace_addr = mem->userspace_addr;
  289. }
  290. if (npages && !new.lpage_info) {
  291. int largepages = npages / KVM_PAGES_PER_HPAGE;
  292. if (npages % KVM_PAGES_PER_HPAGE)
  293. largepages++;
  294. if (base_gfn % KVM_PAGES_PER_HPAGE)
  295. largepages++;
  296. new.lpage_info = vmalloc(largepages * sizeof(*new.lpage_info));
  297. if (!new.lpage_info)
  298. goto out_free;
  299. memset(new.lpage_info, 0, largepages * sizeof(*new.lpage_info));
  300. if (base_gfn % KVM_PAGES_PER_HPAGE)
  301. new.lpage_info[0].write_count = 1;
  302. if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE)
  303. new.lpage_info[largepages-1].write_count = 1;
  304. }
  305. /* Allocate page dirty bitmap if needed */
  306. if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
  307. unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
  308. new.dirty_bitmap = vmalloc(dirty_bytes);
  309. if (!new.dirty_bitmap)
  310. goto out_free;
  311. memset(new.dirty_bitmap, 0, dirty_bytes);
  312. }
  313. if (mem->slot >= kvm->nmemslots)
  314. kvm->nmemslots = mem->slot + 1;
  315. *memslot = new;
  316. r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
  317. if (r) {
  318. *memslot = old;
  319. goto out_free;
  320. }
  321. kvm_free_physmem_slot(&old, &new);
  322. return 0;
  323. out_free:
  324. kvm_free_physmem_slot(&new, &old);
  325. out:
  326. return r;
  327. }
  328. EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
  329. int kvm_set_memory_region(struct kvm *kvm,
  330. struct kvm_userspace_memory_region *mem,
  331. int user_alloc)
  332. {
  333. int r;
  334. down_write(&kvm->slots_lock);
  335. r = __kvm_set_memory_region(kvm, mem, user_alloc);
  336. up_write(&kvm->slots_lock);
  337. return r;
  338. }
  339. EXPORT_SYMBOL_GPL(kvm_set_memory_region);
  340. int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  341. struct
  342. kvm_userspace_memory_region *mem,
  343. int user_alloc)
  344. {
  345. if (mem->slot >= KVM_MEMORY_SLOTS)
  346. return -EINVAL;
  347. return kvm_set_memory_region(kvm, mem, user_alloc);
  348. }
  349. int kvm_get_dirty_log(struct kvm *kvm,
  350. struct kvm_dirty_log *log, int *is_dirty)
  351. {
  352. struct kvm_memory_slot *memslot;
  353. int r, i;
  354. int n;
  355. unsigned long any = 0;
  356. r = -EINVAL;
  357. if (log->slot >= KVM_MEMORY_SLOTS)
  358. goto out;
  359. memslot = &kvm->memslots[log->slot];
  360. r = -ENOENT;
  361. if (!memslot->dirty_bitmap)
  362. goto out;
  363. n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
  364. for (i = 0; !any && i < n/sizeof(long); ++i)
  365. any = memslot->dirty_bitmap[i];
  366. r = -EFAULT;
  367. if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
  368. goto out;
  369. if (any)
  370. *is_dirty = 1;
  371. r = 0;
  372. out:
  373. return r;
  374. }
  375. int is_error_page(struct page *page)
  376. {
  377. return page == bad_page;
  378. }
  379. EXPORT_SYMBOL_GPL(is_error_page);
  380. int is_error_pfn(pfn_t pfn)
  381. {
  382. return pfn == bad_pfn;
  383. }
  384. EXPORT_SYMBOL_GPL(is_error_pfn);
  385. static inline unsigned long bad_hva(void)
  386. {
  387. return PAGE_OFFSET;
  388. }
  389. int kvm_is_error_hva(unsigned long addr)
  390. {
  391. return addr == bad_hva();
  392. }
  393. EXPORT_SYMBOL_GPL(kvm_is_error_hva);
  394. static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  395. {
  396. int i;
  397. for (i = 0; i < kvm->nmemslots; ++i) {
  398. struct kvm_memory_slot *memslot = &kvm->memslots[i];
  399. if (gfn >= memslot->base_gfn
  400. && gfn < memslot->base_gfn + memslot->npages)
  401. return memslot;
  402. }
  403. return NULL;
  404. }
  405. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  406. {
  407. gfn = unalias_gfn(kvm, gfn);
  408. return __gfn_to_memslot(kvm, gfn);
  409. }
  410. int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
  411. {
  412. int i;
  413. gfn = unalias_gfn(kvm, gfn);
  414. for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
  415. struct kvm_memory_slot *memslot = &kvm->memslots[i];
  416. if (gfn >= memslot->base_gfn
  417. && gfn < memslot->base_gfn + memslot->npages)
  418. return 1;
  419. }
  420. return 0;
  421. }
  422. EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
  423. unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
  424. {
  425. struct kvm_memory_slot *slot;
  426. gfn = unalias_gfn(kvm, gfn);
  427. slot = __gfn_to_memslot(kvm, gfn);
  428. if (!slot)
  429. return bad_hva();
  430. return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
  431. }
  432. /*
  433. * Requires current->mm->mmap_sem to be held
  434. */
  435. pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
  436. {
  437. struct page *page[1];
  438. unsigned long addr;
  439. int npages;
  440. might_sleep();
  441. addr = gfn_to_hva(kvm, gfn);
  442. if (kvm_is_error_hva(addr)) {
  443. get_page(bad_page);
  444. return page_to_pfn(bad_page);
  445. }
  446. npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page,
  447. NULL);
  448. if (npages != 1) {
  449. get_page(bad_page);
  450. return page_to_pfn(bad_page);
  451. }
  452. return page_to_pfn(page[0]);
  453. }
  454. EXPORT_SYMBOL_GPL(gfn_to_pfn);
  455. struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
  456. {
  457. return pfn_to_page(gfn_to_pfn(kvm, gfn));
  458. }
  459. EXPORT_SYMBOL_GPL(gfn_to_page);
  460. void kvm_release_page_clean(struct page *page)
  461. {
  462. kvm_release_pfn_clean(page_to_pfn(page));
  463. }
  464. EXPORT_SYMBOL_GPL(kvm_release_page_clean);
  465. void kvm_release_pfn_clean(pfn_t pfn)
  466. {
  467. put_page(pfn_to_page(pfn));
  468. }
  469. EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
  470. void kvm_release_page_dirty(struct page *page)
  471. {
  472. kvm_release_pfn_dirty(page_to_pfn(page));
  473. }
  474. EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
  475. void kvm_release_pfn_dirty(pfn_t pfn)
  476. {
  477. kvm_set_pfn_dirty(pfn);
  478. kvm_release_pfn_clean(pfn);
  479. }
  480. EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
  481. void kvm_set_page_dirty(struct page *page)
  482. {
  483. kvm_set_pfn_dirty(page_to_pfn(page));
  484. }
  485. EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
  486. void kvm_set_pfn_dirty(pfn_t pfn)
  487. {
  488. struct page *page = pfn_to_page(pfn);
  489. if (!PageReserved(page))
  490. SetPageDirty(page);
  491. }
  492. EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
  493. void kvm_set_pfn_accessed(pfn_t pfn)
  494. {
  495. mark_page_accessed(pfn_to_page(pfn));
  496. }
  497. EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
  498. void kvm_get_pfn(pfn_t pfn)
  499. {
  500. get_page(pfn_to_page(pfn));
  501. }
  502. EXPORT_SYMBOL_GPL(kvm_get_pfn);
  503. static int next_segment(unsigned long len, int offset)
  504. {
  505. if (len > PAGE_SIZE - offset)
  506. return PAGE_SIZE - offset;
  507. else
  508. return len;
  509. }
  510. int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
  511. int len)
  512. {
  513. int r;
  514. unsigned long addr;
  515. addr = gfn_to_hva(kvm, gfn);
  516. if (kvm_is_error_hva(addr))
  517. return -EFAULT;
  518. r = copy_from_user(data, (void __user *)addr + offset, len);
  519. if (r)
  520. return -EFAULT;
  521. return 0;
  522. }
  523. EXPORT_SYMBOL_GPL(kvm_read_guest_page);
  524. int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
  525. {
  526. gfn_t gfn = gpa >> PAGE_SHIFT;
  527. int seg;
  528. int offset = offset_in_page(gpa);
  529. int ret;
  530. while ((seg = next_segment(len, offset)) != 0) {
  531. ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
  532. if (ret < 0)
  533. return ret;
  534. offset = 0;
  535. len -= seg;
  536. data += seg;
  537. ++gfn;
  538. }
  539. return 0;
  540. }
  541. EXPORT_SYMBOL_GPL(kvm_read_guest);
  542. int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
  543. unsigned long len)
  544. {
  545. int r;
  546. unsigned long addr;
  547. gfn_t gfn = gpa >> PAGE_SHIFT;
  548. int offset = offset_in_page(gpa);
  549. addr = gfn_to_hva(kvm, gfn);
  550. if (kvm_is_error_hva(addr))
  551. return -EFAULT;
  552. pagefault_disable();
  553. r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
  554. pagefault_enable();
  555. if (r)
  556. return -EFAULT;
  557. return 0;
  558. }
  559. EXPORT_SYMBOL(kvm_read_guest_atomic);
  560. int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
  561. int offset, int len)
  562. {
  563. int r;
  564. unsigned long addr;
  565. addr = gfn_to_hva(kvm, gfn);
  566. if (kvm_is_error_hva(addr))
  567. return -EFAULT;
  568. r = copy_to_user((void __user *)addr + offset, data, len);
  569. if (r)
  570. return -EFAULT;
  571. mark_page_dirty(kvm, gfn);
  572. return 0;
  573. }
  574. EXPORT_SYMBOL_GPL(kvm_write_guest_page);
  575. int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
  576. unsigned long len)
  577. {
  578. gfn_t gfn = gpa >> PAGE_SHIFT;
  579. int seg;
  580. int offset = offset_in_page(gpa);
  581. int ret;
  582. while ((seg = next_segment(len, offset)) != 0) {
  583. ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
  584. if (ret < 0)
  585. return ret;
  586. offset = 0;
  587. len -= seg;
  588. data += seg;
  589. ++gfn;
  590. }
  591. return 0;
  592. }
  593. int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
  594. {
  595. return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
  596. }
  597. EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
  598. int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
  599. {
  600. gfn_t gfn = gpa >> PAGE_SHIFT;
  601. int seg;
  602. int offset = offset_in_page(gpa);
  603. int ret;
  604. while ((seg = next_segment(len, offset)) != 0) {
  605. ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
  606. if (ret < 0)
  607. return ret;
  608. offset = 0;
  609. len -= seg;
  610. ++gfn;
  611. }
  612. return 0;
  613. }
  614. EXPORT_SYMBOL_GPL(kvm_clear_guest);
  615. void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
  616. {
  617. struct kvm_memory_slot *memslot;
  618. gfn = unalias_gfn(kvm, gfn);
  619. memslot = __gfn_to_memslot(kvm, gfn);
  620. if (memslot && memslot->dirty_bitmap) {
  621. unsigned long rel_gfn = gfn - memslot->base_gfn;
  622. /* avoid RMW */
  623. if (!test_bit(rel_gfn, memslot->dirty_bitmap))
  624. set_bit(rel_gfn, memslot->dirty_bitmap);
  625. }
  626. }
  627. /*
  628. * The vCPU has executed a HLT instruction with in-kernel mode enabled.
  629. */
  630. void kvm_vcpu_block(struct kvm_vcpu *vcpu)
  631. {
  632. DECLARE_WAITQUEUE(wait, current);
  633. add_wait_queue(&vcpu->wq, &wait);
  634. /*
  635. * We will block until either an interrupt or a signal wakes us up
  636. */
  637. while (!kvm_cpu_has_interrupt(vcpu)
  638. && !kvm_cpu_has_pending_timer(vcpu)
  639. && !signal_pending(current)
  640. && !kvm_arch_vcpu_runnable(vcpu)) {
  641. set_current_state(TASK_INTERRUPTIBLE);
  642. vcpu_put(vcpu);
  643. schedule();
  644. vcpu_load(vcpu);
  645. }
  646. __set_current_state(TASK_RUNNING);
  647. remove_wait_queue(&vcpu->wq, &wait);
  648. }
  649. void kvm_resched(struct kvm_vcpu *vcpu)
  650. {
  651. if (!need_resched())
  652. return;
  653. cond_resched();
  654. }
  655. EXPORT_SYMBOL_GPL(kvm_resched);
  656. static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  657. {
  658. struct kvm_vcpu *vcpu = vma->vm_file->private_data;
  659. struct page *page;
  660. if (vmf->pgoff == 0)
  661. page = virt_to_page(vcpu->run);
  662. #ifdef CONFIG_X86
  663. else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
  664. page = virt_to_page(vcpu->arch.pio_data);
  665. #endif
  666. else
  667. return VM_FAULT_SIGBUS;
  668. get_page(page);
  669. vmf->page = page;
  670. return 0;
  671. }
  672. static struct vm_operations_struct kvm_vcpu_vm_ops = {
  673. .fault = kvm_vcpu_fault,
  674. };
  675. static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
  676. {
  677. vma->vm_ops = &kvm_vcpu_vm_ops;
  678. return 0;
  679. }
  680. static int kvm_vcpu_release(struct inode *inode, struct file *filp)
  681. {
  682. struct kvm_vcpu *vcpu = filp->private_data;
  683. kvm_put_kvm(vcpu->kvm);
  684. return 0;
  685. }
  686. static const struct file_operations kvm_vcpu_fops = {
  687. .release = kvm_vcpu_release,
  688. .unlocked_ioctl = kvm_vcpu_ioctl,
  689. .compat_ioctl = kvm_vcpu_ioctl,
  690. .mmap = kvm_vcpu_mmap,
  691. };
  692. /*
  693. * Allocates an inode for the vcpu.
  694. */
  695. static int create_vcpu_fd(struct kvm_vcpu *vcpu)
  696. {
  697. int fd, r;
  698. struct inode *inode;
  699. struct file *file;
  700. r = anon_inode_getfd(&fd, &inode, &file,
  701. "kvm-vcpu", &kvm_vcpu_fops, vcpu);
  702. if (r) {
  703. kvm_put_kvm(vcpu->kvm);
  704. return r;
  705. }
  706. return fd;
  707. }
  708. /*
  709. * Creates some virtual cpus. Good luck creating more than one.
  710. */
  711. static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
  712. {
  713. int r;
  714. struct kvm_vcpu *vcpu;
  715. if (!valid_vcpu(n))
  716. return -EINVAL;
  717. vcpu = kvm_arch_vcpu_create(kvm, n);
  718. if (IS_ERR(vcpu))
  719. return PTR_ERR(vcpu);
  720. preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
  721. r = kvm_arch_vcpu_setup(vcpu);
  722. if (r)
  723. goto vcpu_destroy;
  724. mutex_lock(&kvm->lock);
  725. if (kvm->vcpus[n]) {
  726. r = -EEXIST;
  727. mutex_unlock(&kvm->lock);
  728. goto vcpu_destroy;
  729. }
  730. kvm->vcpus[n] = vcpu;
  731. mutex_unlock(&kvm->lock);
  732. /* Now it's all set up, let userspace reach it */
  733. kvm_get_kvm(kvm);
  734. r = create_vcpu_fd(vcpu);
  735. if (r < 0)
  736. goto unlink;
  737. return r;
  738. unlink:
  739. mutex_lock(&kvm->lock);
  740. kvm->vcpus[n] = NULL;
  741. mutex_unlock(&kvm->lock);
  742. vcpu_destroy:
  743. kvm_arch_vcpu_destroy(vcpu);
  744. return r;
  745. }
  746. static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
  747. {
  748. if (sigset) {
  749. sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
  750. vcpu->sigset_active = 1;
  751. vcpu->sigset = *sigset;
  752. } else
  753. vcpu->sigset_active = 0;
  754. return 0;
  755. }
  756. static long kvm_vcpu_ioctl(struct file *filp,
  757. unsigned int ioctl, unsigned long arg)
  758. {
  759. struct kvm_vcpu *vcpu = filp->private_data;
  760. void __user *argp = (void __user *)arg;
  761. int r;
  762. if (vcpu->kvm->mm != current->mm)
  763. return -EIO;
  764. switch (ioctl) {
  765. case KVM_RUN:
  766. r = -EINVAL;
  767. if (arg)
  768. goto out;
  769. r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
  770. break;
  771. case KVM_GET_REGS: {
  772. struct kvm_regs *kvm_regs;
  773. r = -ENOMEM;
  774. kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
  775. if (!kvm_regs)
  776. goto out;
  777. r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
  778. if (r)
  779. goto out_free1;
  780. r = -EFAULT;
  781. if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
  782. goto out_free1;
  783. r = 0;
  784. out_free1:
  785. kfree(kvm_regs);
  786. break;
  787. }
  788. case KVM_SET_REGS: {
  789. struct kvm_regs *kvm_regs;
  790. r = -ENOMEM;
  791. kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
  792. if (!kvm_regs)
  793. goto out;
  794. r = -EFAULT;
  795. if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
  796. goto out_free2;
  797. r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
  798. if (r)
  799. goto out_free2;
  800. r = 0;
  801. out_free2:
  802. kfree(kvm_regs);
  803. break;
  804. }
  805. case KVM_GET_SREGS: {
  806. struct kvm_sregs kvm_sregs;
  807. memset(&kvm_sregs, 0, sizeof kvm_sregs);
  808. r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
  809. if (r)
  810. goto out;
  811. r = -EFAULT;
  812. if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
  813. goto out;
  814. r = 0;
  815. break;
  816. }
  817. case KVM_SET_SREGS: {
  818. struct kvm_sregs kvm_sregs;
  819. r = -EFAULT;
  820. if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
  821. goto out;
  822. r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
  823. if (r)
  824. goto out;
  825. r = 0;
  826. break;
  827. }
  828. case KVM_GET_MP_STATE: {
  829. struct kvm_mp_state mp_state;
  830. r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
  831. if (r)
  832. goto out;
  833. r = -EFAULT;
  834. if (copy_to_user(argp, &mp_state, sizeof mp_state))
  835. goto out;
  836. r = 0;
  837. break;
  838. }
  839. case KVM_SET_MP_STATE: {
  840. struct kvm_mp_state mp_state;
  841. r = -EFAULT;
  842. if (copy_from_user(&mp_state, argp, sizeof mp_state))
  843. goto out;
  844. r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
  845. if (r)
  846. goto out;
  847. r = 0;
  848. break;
  849. }
  850. case KVM_TRANSLATE: {
  851. struct kvm_translation tr;
  852. r = -EFAULT;
  853. if (copy_from_user(&tr, argp, sizeof tr))
  854. goto out;
  855. r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
  856. if (r)
  857. goto out;
  858. r = -EFAULT;
  859. if (copy_to_user(argp, &tr, sizeof tr))
  860. goto out;
  861. r = 0;
  862. break;
  863. }
  864. case KVM_DEBUG_GUEST: {
  865. struct kvm_debug_guest dbg;
  866. r = -EFAULT;
  867. if (copy_from_user(&dbg, argp, sizeof dbg))
  868. goto out;
  869. r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
  870. if (r)
  871. goto out;
  872. r = 0;
  873. break;
  874. }
  875. case KVM_SET_SIGNAL_MASK: {
  876. struct kvm_signal_mask __user *sigmask_arg = argp;
  877. struct kvm_signal_mask kvm_sigmask;
  878. sigset_t sigset, *p;
  879. p = NULL;
  880. if (argp) {
  881. r = -EFAULT;
  882. if (copy_from_user(&kvm_sigmask, argp,
  883. sizeof kvm_sigmask))
  884. goto out;
  885. r = -EINVAL;
  886. if (kvm_sigmask.len != sizeof sigset)
  887. goto out;
  888. r = -EFAULT;
  889. if (copy_from_user(&sigset, sigmask_arg->sigset,
  890. sizeof sigset))
  891. goto out;
  892. p = &sigset;
  893. }
  894. r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
  895. break;
  896. }
  897. case KVM_GET_FPU: {
  898. struct kvm_fpu fpu;
  899. memset(&fpu, 0, sizeof fpu);
  900. r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu);
  901. if (r)
  902. goto out;
  903. r = -EFAULT;
  904. if (copy_to_user(argp, &fpu, sizeof fpu))
  905. goto out;
  906. r = 0;
  907. break;
  908. }
  909. case KVM_SET_FPU: {
  910. struct kvm_fpu fpu;
  911. r = -EFAULT;
  912. if (copy_from_user(&fpu, argp, sizeof fpu))
  913. goto out;
  914. r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu);
  915. if (r)
  916. goto out;
  917. r = 0;
  918. break;
  919. }
  920. default:
  921. r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  922. }
  923. out:
  924. return r;
  925. }
  926. static long kvm_vm_ioctl(struct file *filp,
  927. unsigned int ioctl, unsigned long arg)
  928. {
  929. struct kvm *kvm = filp->private_data;
  930. void __user *argp = (void __user *)arg;
  931. int r;
  932. if (kvm->mm != current->mm)
  933. return -EIO;
  934. switch (ioctl) {
  935. case KVM_CREATE_VCPU:
  936. r = kvm_vm_ioctl_create_vcpu(kvm, arg);
  937. if (r < 0)
  938. goto out;
  939. break;
  940. case KVM_SET_USER_MEMORY_REGION: {
  941. struct kvm_userspace_memory_region kvm_userspace_mem;
  942. r = -EFAULT;
  943. if (copy_from_user(&kvm_userspace_mem, argp,
  944. sizeof kvm_userspace_mem))
  945. goto out;
  946. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
  947. if (r)
  948. goto out;
  949. break;
  950. }
  951. case KVM_GET_DIRTY_LOG: {
  952. struct kvm_dirty_log log;
  953. r = -EFAULT;
  954. if (copy_from_user(&log, argp, sizeof log))
  955. goto out;
  956. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  957. if (r)
  958. goto out;
  959. break;
  960. }
  961. default:
  962. r = kvm_arch_vm_ioctl(filp, ioctl, arg);
  963. }
  964. out:
  965. return r;
  966. }
  967. static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  968. {
  969. struct kvm *kvm = vma->vm_file->private_data;
  970. struct page *page;
  971. if (!kvm_is_visible_gfn(kvm, vmf->pgoff))
  972. return VM_FAULT_SIGBUS;
  973. page = gfn_to_page(kvm, vmf->pgoff);
  974. if (is_error_page(page)) {
  975. kvm_release_page_clean(page);
  976. return VM_FAULT_SIGBUS;
  977. }
  978. vmf->page = page;
  979. return 0;
  980. }
  981. static struct vm_operations_struct kvm_vm_vm_ops = {
  982. .fault = kvm_vm_fault,
  983. };
  984. static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
  985. {
  986. vma->vm_ops = &kvm_vm_vm_ops;
  987. return 0;
  988. }
  989. static const struct file_operations kvm_vm_fops = {
  990. .release = kvm_vm_release,
  991. .unlocked_ioctl = kvm_vm_ioctl,
  992. .compat_ioctl = kvm_vm_ioctl,
  993. .mmap = kvm_vm_mmap,
  994. };
  995. static int kvm_dev_ioctl_create_vm(void)
  996. {
  997. int fd, r;
  998. struct inode *inode;
  999. struct file *file;
  1000. struct kvm *kvm;
  1001. kvm = kvm_create_vm();
  1002. if (IS_ERR(kvm))
  1003. return PTR_ERR(kvm);
  1004. r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
  1005. if (r) {
  1006. kvm_put_kvm(kvm);
  1007. return r;
  1008. }
  1009. return fd;
  1010. }
  1011. static long kvm_dev_ioctl(struct file *filp,
  1012. unsigned int ioctl, unsigned long arg)
  1013. {
  1014. void __user *argp = (void __user *)arg;
  1015. long r = -EINVAL;
  1016. switch (ioctl) {
  1017. case KVM_GET_API_VERSION:
  1018. r = -EINVAL;
  1019. if (arg)
  1020. goto out;
  1021. r = KVM_API_VERSION;
  1022. break;
  1023. case KVM_CREATE_VM:
  1024. r = -EINVAL;
  1025. if (arg)
  1026. goto out;
  1027. r = kvm_dev_ioctl_create_vm();
  1028. break;
  1029. case KVM_CHECK_EXTENSION:
  1030. r = kvm_dev_ioctl_check_extension((long)argp);
  1031. break;
  1032. case KVM_GET_VCPU_MMAP_SIZE:
  1033. r = -EINVAL;
  1034. if (arg)
  1035. goto out;
  1036. r = PAGE_SIZE; /* struct kvm_run */
  1037. #ifdef CONFIG_X86
  1038. r += PAGE_SIZE; /* pio data page */
  1039. #endif
  1040. break;
  1041. case KVM_TRACE_ENABLE:
  1042. case KVM_TRACE_PAUSE:
  1043. case KVM_TRACE_DISABLE:
  1044. r = kvm_trace_ioctl(ioctl, arg);
  1045. break;
  1046. default:
  1047. return kvm_arch_dev_ioctl(filp, ioctl, arg);
  1048. }
  1049. out:
  1050. return r;
  1051. }
  1052. static struct file_operations kvm_chardev_ops = {
  1053. .unlocked_ioctl = kvm_dev_ioctl,
  1054. .compat_ioctl = kvm_dev_ioctl,
  1055. };
  1056. static struct miscdevice kvm_dev = {
  1057. KVM_MINOR,
  1058. "kvm",
  1059. &kvm_chardev_ops,
  1060. };
  1061. static void hardware_enable(void *junk)
  1062. {
  1063. int cpu = raw_smp_processor_id();
  1064. if (cpu_isset(cpu, cpus_hardware_enabled))
  1065. return;
  1066. cpu_set(cpu, cpus_hardware_enabled);
  1067. kvm_arch_hardware_enable(NULL);
  1068. }
  1069. static void hardware_disable(void *junk)
  1070. {
  1071. int cpu = raw_smp_processor_id();
  1072. if (!cpu_isset(cpu, cpus_hardware_enabled))
  1073. return;
  1074. cpu_clear(cpu, cpus_hardware_enabled);
  1075. decache_vcpus_on_cpu(cpu);
  1076. kvm_arch_hardware_disable(NULL);
  1077. }
  1078. static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  1079. void *v)
  1080. {
  1081. int cpu = (long)v;
  1082. val &= ~CPU_TASKS_FROZEN;
  1083. switch (val) {
  1084. case CPU_DYING:
  1085. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  1086. cpu);
  1087. hardware_disable(NULL);
  1088. break;
  1089. case CPU_UP_CANCELED:
  1090. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  1091. cpu);
  1092. smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
  1093. break;
  1094. case CPU_ONLINE:
  1095. printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
  1096. cpu);
  1097. smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
  1098. break;
  1099. }
  1100. return NOTIFY_OK;
  1101. }
  1102. static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
  1103. void *v)
  1104. {
  1105. if (val == SYS_RESTART) {
  1106. /*
  1107. * Some (well, at least mine) BIOSes hang on reboot if
  1108. * in vmx root mode.
  1109. */
  1110. printk(KERN_INFO "kvm: exiting hardware virtualization\n");
  1111. on_each_cpu(hardware_disable, NULL, 0, 1);
  1112. }
  1113. return NOTIFY_OK;
  1114. }
  1115. static struct notifier_block kvm_reboot_notifier = {
  1116. .notifier_call = kvm_reboot,
  1117. .priority = 0,
  1118. };
  1119. void kvm_io_bus_init(struct kvm_io_bus *bus)
  1120. {
  1121. memset(bus, 0, sizeof(*bus));
  1122. }
  1123. void kvm_io_bus_destroy(struct kvm_io_bus *bus)
  1124. {
  1125. int i;
  1126. for (i = 0; i < bus->dev_count; i++) {
  1127. struct kvm_io_device *pos = bus->devs[i];
  1128. kvm_iodevice_destructor(pos);
  1129. }
  1130. }
  1131. struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
  1132. {
  1133. int i;
  1134. for (i = 0; i < bus->dev_count; i++) {
  1135. struct kvm_io_device *pos = bus->devs[i];
  1136. if (pos->in_range(pos, addr))
  1137. return pos;
  1138. }
  1139. return NULL;
  1140. }
  1141. void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
  1142. {
  1143. BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
  1144. bus->devs[bus->dev_count++] = dev;
  1145. }
  1146. static struct notifier_block kvm_cpu_notifier = {
  1147. .notifier_call = kvm_cpu_hotplug,
  1148. .priority = 20, /* must be > scheduler priority */
  1149. };
  1150. static int vm_stat_get(void *_offset, u64 *val)
  1151. {
  1152. unsigned offset = (long)_offset;
  1153. struct kvm *kvm;
  1154. *val = 0;
  1155. spin_lock(&kvm_lock);
  1156. list_for_each_entry(kvm, &vm_list, vm_list)
  1157. *val += *(u32 *)((void *)kvm + offset);
  1158. spin_unlock(&kvm_lock);
  1159. return 0;
  1160. }
  1161. DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
  1162. static int vcpu_stat_get(void *_offset, u64 *val)
  1163. {
  1164. unsigned offset = (long)_offset;
  1165. struct kvm *kvm;
  1166. struct kvm_vcpu *vcpu;
  1167. int i;
  1168. *val = 0;
  1169. spin_lock(&kvm_lock);
  1170. list_for_each_entry(kvm, &vm_list, vm_list)
  1171. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  1172. vcpu = kvm->vcpus[i];
  1173. if (vcpu)
  1174. *val += *(u32 *)((void *)vcpu + offset);
  1175. }
  1176. spin_unlock(&kvm_lock);
  1177. return 0;
  1178. }
  1179. DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
  1180. static struct file_operations *stat_fops[] = {
  1181. [KVM_STAT_VCPU] = &vcpu_stat_fops,
  1182. [KVM_STAT_VM] = &vm_stat_fops,
  1183. };
  1184. static void kvm_init_debug(void)
  1185. {
  1186. struct kvm_stats_debugfs_item *p;
  1187. kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
  1188. for (p = debugfs_entries; p->name; ++p)
  1189. p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
  1190. (void *)(long)p->offset,
  1191. stat_fops[p->kind]);
  1192. }
  1193. static void kvm_exit_debug(void)
  1194. {
  1195. struct kvm_stats_debugfs_item *p;
  1196. for (p = debugfs_entries; p->name; ++p)
  1197. debugfs_remove(p->dentry);
  1198. debugfs_remove(kvm_debugfs_dir);
  1199. }
  1200. static int kvm_suspend(struct sys_device *dev, pm_message_t state)
  1201. {
  1202. hardware_disable(NULL);
  1203. return 0;
  1204. }
  1205. static int kvm_resume(struct sys_device *dev)
  1206. {
  1207. hardware_enable(NULL);
  1208. return 0;
  1209. }
  1210. static struct sysdev_class kvm_sysdev_class = {
  1211. .name = "kvm",
  1212. .suspend = kvm_suspend,
  1213. .resume = kvm_resume,
  1214. };
  1215. static struct sys_device kvm_sysdev = {
  1216. .id = 0,
  1217. .cls = &kvm_sysdev_class,
  1218. };
  1219. struct page *bad_page;
  1220. pfn_t bad_pfn;
  1221. static inline
  1222. struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
  1223. {
  1224. return container_of(pn, struct kvm_vcpu, preempt_notifier);
  1225. }
  1226. static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
  1227. {
  1228. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  1229. kvm_arch_vcpu_load(vcpu, cpu);
  1230. }
  1231. static void kvm_sched_out(struct preempt_notifier *pn,
  1232. struct task_struct *next)
  1233. {
  1234. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  1235. kvm_arch_vcpu_put(vcpu);
  1236. }
  1237. int kvm_init(void *opaque, unsigned int vcpu_size,
  1238. struct module *module)
  1239. {
  1240. int r;
  1241. int cpu;
  1242. kvm_init_debug();
  1243. r = kvm_arch_init(opaque);
  1244. if (r)
  1245. goto out_fail;
  1246. bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  1247. if (bad_page == NULL) {
  1248. r = -ENOMEM;
  1249. goto out;
  1250. }
  1251. bad_pfn = page_to_pfn(bad_page);
  1252. r = kvm_arch_hardware_setup();
  1253. if (r < 0)
  1254. goto out_free_0;
  1255. for_each_online_cpu(cpu) {
  1256. smp_call_function_single(cpu,
  1257. kvm_arch_check_processor_compat,
  1258. &r, 0, 1);
  1259. if (r < 0)
  1260. goto out_free_1;
  1261. }
  1262. on_each_cpu(hardware_enable, NULL, 0, 1);
  1263. r = register_cpu_notifier(&kvm_cpu_notifier);
  1264. if (r)
  1265. goto out_free_2;
  1266. register_reboot_notifier(&kvm_reboot_notifier);
  1267. r = sysdev_class_register(&kvm_sysdev_class);
  1268. if (r)
  1269. goto out_free_3;
  1270. r = sysdev_register(&kvm_sysdev);
  1271. if (r)
  1272. goto out_free_4;
  1273. /* A kmem cache lets us meet the alignment requirements of fx_save. */
  1274. kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
  1275. __alignof__(struct kvm_vcpu),
  1276. 0, NULL);
  1277. if (!kvm_vcpu_cache) {
  1278. r = -ENOMEM;
  1279. goto out_free_5;
  1280. }
  1281. kvm_chardev_ops.owner = module;
  1282. r = misc_register(&kvm_dev);
  1283. if (r) {
  1284. printk(KERN_ERR "kvm: misc device register failed\n");
  1285. goto out_free;
  1286. }
  1287. kvm_preempt_ops.sched_in = kvm_sched_in;
  1288. kvm_preempt_ops.sched_out = kvm_sched_out;
  1289. return 0;
  1290. out_free:
  1291. kmem_cache_destroy(kvm_vcpu_cache);
  1292. out_free_5:
  1293. sysdev_unregister(&kvm_sysdev);
  1294. out_free_4:
  1295. sysdev_class_unregister(&kvm_sysdev_class);
  1296. out_free_3:
  1297. unregister_reboot_notifier(&kvm_reboot_notifier);
  1298. unregister_cpu_notifier(&kvm_cpu_notifier);
  1299. out_free_2:
  1300. on_each_cpu(hardware_disable, NULL, 0, 1);
  1301. out_free_1:
  1302. kvm_arch_hardware_unsetup();
  1303. out_free_0:
  1304. __free_page(bad_page);
  1305. out:
  1306. kvm_arch_exit();
  1307. kvm_exit_debug();
  1308. out_fail:
  1309. return r;
  1310. }
  1311. EXPORT_SYMBOL_GPL(kvm_init);
  1312. void kvm_exit(void)
  1313. {
  1314. kvm_trace_cleanup();
  1315. misc_deregister(&kvm_dev);
  1316. kmem_cache_destroy(kvm_vcpu_cache);
  1317. sysdev_unregister(&kvm_sysdev);
  1318. sysdev_class_unregister(&kvm_sysdev_class);
  1319. unregister_reboot_notifier(&kvm_reboot_notifier);
  1320. unregister_cpu_notifier(&kvm_cpu_notifier);
  1321. on_each_cpu(hardware_disable, NULL, 0, 1);
  1322. kvm_arch_hardware_unsetup();
  1323. kvm_arch_exit();
  1324. kvm_exit_debug();
  1325. __free_page(bad_page);
  1326. }
  1327. EXPORT_SYMBOL_GPL(kvm_exit);