kvm_main.c 72 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  9. *
  10. * Authors:
  11. * Avi Kivity <avi@qumranet.com>
  12. * Yaniv Kamay <yaniv@qumranet.com>
  13. *
  14. * This work is licensed under the terms of the GNU GPL, version 2. See
  15. * the COPYING file in the top-level directory.
  16. *
  17. */
  18. #include "iodev.h"
  19. #include <linux/kvm_host.h>
  20. #include <linux/kvm.h>
  21. #include <linux/module.h>
  22. #include <linux/errno.h>
  23. #include <linux/percpu.h>
  24. #include <linux/mm.h>
  25. #include <linux/miscdevice.h>
  26. #include <linux/vmalloc.h>
  27. #include <linux/reboot.h>
  28. #include <linux/debugfs.h>
  29. #include <linux/highmem.h>
  30. #include <linux/file.h>
  31. #include <linux/syscore_ops.h>
  32. #include <linux/cpu.h>
  33. #include <linux/sched.h>
  34. #include <linux/cpumask.h>
  35. #include <linux/smp.h>
  36. #include <linux/anon_inodes.h>
  37. #include <linux/profile.h>
  38. #include <linux/kvm_para.h>
  39. #include <linux/pagemap.h>
  40. #include <linux/mman.h>
  41. #include <linux/swap.h>
  42. #include <linux/bitops.h>
  43. #include <linux/spinlock.h>
  44. #include <linux/compat.h>
  45. #include <linux/srcu.h>
  46. #include <linux/hugetlb.h>
  47. #include <linux/slab.h>
  48. #include <linux/sort.h>
  49. #include <linux/bsearch.h>
  50. #include <asm/processor.h>
  51. #include <asm/io.h>
  52. #include <asm/uaccess.h>
  53. #include <asm/pgtable.h>
  54. #include "coalesced_mmio.h"
  55. #include "async_pf.h"
  56. #define CREATE_TRACE_POINTS
  57. #include <trace/events/kvm.h>
  58. MODULE_AUTHOR("Qumranet");
  59. MODULE_LICENSE("GPL");
  60. /*
  61. * Ordering of locks:
  62. *
  63. * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
  64. */
  65. DEFINE_RAW_SPINLOCK(kvm_lock);
  66. LIST_HEAD(vm_list);
  67. static cpumask_var_t cpus_hardware_enabled;
  68. static int kvm_usage_count = 0;
  69. static atomic_t hardware_enable_failed;
  70. struct kmem_cache *kvm_vcpu_cache;
  71. EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  72. static __read_mostly struct preempt_ops kvm_preempt_ops;
  73. struct dentry *kvm_debugfs_dir;
  74. static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  75. unsigned long arg);
  76. #ifdef CONFIG_COMPAT
  77. static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
  78. unsigned long arg);
  79. #endif
  80. static int hardware_enable_all(void);
  81. static void hardware_disable_all(void);
  82. static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
  83. bool kvm_rebooting;
  84. EXPORT_SYMBOL_GPL(kvm_rebooting);
  85. static bool largepages_enabled = true;
  86. bool kvm_is_mmio_pfn(pfn_t pfn)
  87. {
  88. if (pfn_valid(pfn)) {
  89. int reserved;
  90. struct page *tail = pfn_to_page(pfn);
  91. struct page *head = compound_trans_head(tail);
  92. reserved = PageReserved(head);
  93. if (head != tail) {
  94. /*
  95. * "head" is not a dangling pointer
  96. * (compound_trans_head takes care of that)
  97. * but the hugepage may have been splitted
  98. * from under us (and we may not hold a
  99. * reference count on the head page so it can
  100. * be reused before we run PageReferenced), so
  101. * we've to check PageTail before returning
  102. * what we just read.
  103. */
  104. smp_rmb();
  105. if (PageTail(tail))
  106. return reserved;
  107. }
  108. return PageReserved(tail);
  109. }
  110. return true;
  111. }
  112. /*
  113. * Switches to specified vcpu, until a matching vcpu_put()
  114. */
  115. int vcpu_load(struct kvm_vcpu *vcpu)
  116. {
  117. int cpu;
  118. if (mutex_lock_killable(&vcpu->mutex))
  119. return -EINTR;
  120. if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
  121. /* The thread running this VCPU changed. */
  122. struct pid *oldpid = vcpu->pid;
  123. struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
  124. rcu_assign_pointer(vcpu->pid, newpid);
  125. synchronize_rcu();
  126. put_pid(oldpid);
  127. }
  128. cpu = get_cpu();
  129. preempt_notifier_register(&vcpu->preempt_notifier);
  130. kvm_arch_vcpu_load(vcpu, cpu);
  131. put_cpu();
  132. return 0;
  133. }
  134. void vcpu_put(struct kvm_vcpu *vcpu)
  135. {
  136. preempt_disable();
  137. kvm_arch_vcpu_put(vcpu);
  138. preempt_notifier_unregister(&vcpu->preempt_notifier);
  139. preempt_enable();
  140. mutex_unlock(&vcpu->mutex);
  141. }
  142. static void ack_flush(void *_completed)
  143. {
  144. }
  145. static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
  146. {
  147. int i, cpu, me;
  148. cpumask_var_t cpus;
  149. bool called = true;
  150. struct kvm_vcpu *vcpu;
  151. zalloc_cpumask_var(&cpus, GFP_ATOMIC);
  152. me = get_cpu();
  153. kvm_for_each_vcpu(i, vcpu, kvm) {
  154. kvm_make_request(req, vcpu);
  155. cpu = vcpu->cpu;
  156. /* Set ->requests bit before we read ->mode */
  157. smp_mb();
  158. if (cpus != NULL && cpu != -1 && cpu != me &&
  159. kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
  160. cpumask_set_cpu(cpu, cpus);
  161. }
  162. if (unlikely(cpus == NULL))
  163. smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
  164. else if (!cpumask_empty(cpus))
  165. smp_call_function_many(cpus, ack_flush, NULL, 1);
  166. else
  167. called = false;
  168. put_cpu();
  169. free_cpumask_var(cpus);
  170. return called;
  171. }
  172. void kvm_flush_remote_tlbs(struct kvm *kvm)
  173. {
  174. long dirty_count = kvm->tlbs_dirty;
  175. smp_mb();
  176. if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
  177. ++kvm->stat.remote_tlb_flush;
  178. cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
  179. }
  180. void kvm_reload_remote_mmus(struct kvm *kvm)
  181. {
  182. make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
  183. }
  184. void kvm_make_mclock_inprogress_request(struct kvm *kvm)
  185. {
  186. make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
  187. }
  188. void kvm_make_scan_ioapic_request(struct kvm *kvm)
  189. {
  190. make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
  191. }
  192. int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
  193. {
  194. struct page *page;
  195. int r;
  196. mutex_init(&vcpu->mutex);
  197. vcpu->cpu = -1;
  198. vcpu->kvm = kvm;
  199. vcpu->vcpu_id = id;
  200. vcpu->pid = NULL;
  201. init_waitqueue_head(&vcpu->wq);
  202. kvm_async_pf_vcpu_init(vcpu);
  203. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  204. if (!page) {
  205. r = -ENOMEM;
  206. goto fail;
  207. }
  208. vcpu->run = page_address(page);
  209. kvm_vcpu_set_in_spin_loop(vcpu, false);
  210. kvm_vcpu_set_dy_eligible(vcpu, false);
  211. vcpu->preempted = false;
  212. r = kvm_arch_vcpu_init(vcpu);
  213. if (r < 0)
  214. goto fail_free_run;
  215. return 0;
  216. fail_free_run:
  217. free_page((unsigned long)vcpu->run);
  218. fail:
  219. return r;
  220. }
  221. EXPORT_SYMBOL_GPL(kvm_vcpu_init);
  222. void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
  223. {
  224. put_pid(vcpu->pid);
  225. kvm_arch_vcpu_uninit(vcpu);
  226. free_page((unsigned long)vcpu->run);
  227. }
  228. EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
  229. #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
  230. static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
  231. {
  232. return container_of(mn, struct kvm, mmu_notifier);
  233. }
  234. static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
  235. struct mm_struct *mm,
  236. unsigned long address)
  237. {
  238. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  239. int need_tlb_flush, idx;
  240. /*
  241. * When ->invalidate_page runs, the linux pte has been zapped
  242. * already but the page is still allocated until
  243. * ->invalidate_page returns. So if we increase the sequence
  244. * here the kvm page fault will notice if the spte can't be
  245. * established because the page is going to be freed. If
  246. * instead the kvm page fault establishes the spte before
  247. * ->invalidate_page runs, kvm_unmap_hva will release it
  248. * before returning.
  249. *
  250. * The sequence increase only need to be seen at spin_unlock
  251. * time, and not at spin_lock time.
  252. *
  253. * Increasing the sequence after the spin_unlock would be
  254. * unsafe because the kvm page fault could then establish the
  255. * pte after kvm_unmap_hva returned, without noticing the page
  256. * is going to be freed.
  257. */
  258. idx = srcu_read_lock(&kvm->srcu);
  259. spin_lock(&kvm->mmu_lock);
  260. kvm->mmu_notifier_seq++;
  261. need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
  262. /* we've to flush the tlb before the pages can be freed */
  263. if (need_tlb_flush)
  264. kvm_flush_remote_tlbs(kvm);
  265. spin_unlock(&kvm->mmu_lock);
  266. srcu_read_unlock(&kvm->srcu, idx);
  267. }
  268. static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
  269. struct mm_struct *mm,
  270. unsigned long address,
  271. pte_t pte)
  272. {
  273. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  274. int idx;
  275. idx = srcu_read_lock(&kvm->srcu);
  276. spin_lock(&kvm->mmu_lock);
  277. kvm->mmu_notifier_seq++;
  278. kvm_set_spte_hva(kvm, address, pte);
  279. spin_unlock(&kvm->mmu_lock);
  280. srcu_read_unlock(&kvm->srcu, idx);
  281. }
  282. static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
  283. struct mm_struct *mm,
  284. unsigned long start,
  285. unsigned long end)
  286. {
  287. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  288. int need_tlb_flush = 0, idx;
  289. idx = srcu_read_lock(&kvm->srcu);
  290. spin_lock(&kvm->mmu_lock);
  291. /*
  292. * The count increase must become visible at unlock time as no
  293. * spte can be established without taking the mmu_lock and
  294. * count is also read inside the mmu_lock critical section.
  295. */
  296. kvm->mmu_notifier_count++;
  297. need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
  298. need_tlb_flush |= kvm->tlbs_dirty;
  299. /* we've to flush the tlb before the pages can be freed */
  300. if (need_tlb_flush)
  301. kvm_flush_remote_tlbs(kvm);
  302. spin_unlock(&kvm->mmu_lock);
  303. srcu_read_unlock(&kvm->srcu, idx);
  304. }
  305. static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
  306. struct mm_struct *mm,
  307. unsigned long start,
  308. unsigned long end)
  309. {
  310. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  311. spin_lock(&kvm->mmu_lock);
  312. /*
  313. * This sequence increase will notify the kvm page fault that
  314. * the page that is going to be mapped in the spte could have
  315. * been freed.
  316. */
  317. kvm->mmu_notifier_seq++;
  318. smp_wmb();
  319. /*
  320. * The above sequence increase must be visible before the
  321. * below count decrease, which is ensured by the smp_wmb above
  322. * in conjunction with the smp_rmb in mmu_notifier_retry().
  323. */
  324. kvm->mmu_notifier_count--;
  325. spin_unlock(&kvm->mmu_lock);
  326. BUG_ON(kvm->mmu_notifier_count < 0);
  327. }
  328. static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
  329. struct mm_struct *mm,
  330. unsigned long address)
  331. {
  332. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  333. int young, idx;
  334. idx = srcu_read_lock(&kvm->srcu);
  335. spin_lock(&kvm->mmu_lock);
  336. young = kvm_age_hva(kvm, address);
  337. if (young)
  338. kvm_flush_remote_tlbs(kvm);
  339. spin_unlock(&kvm->mmu_lock);
  340. srcu_read_unlock(&kvm->srcu, idx);
  341. return young;
  342. }
  343. static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
  344. struct mm_struct *mm,
  345. unsigned long address)
  346. {
  347. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  348. int young, idx;
  349. idx = srcu_read_lock(&kvm->srcu);
  350. spin_lock(&kvm->mmu_lock);
  351. young = kvm_test_age_hva(kvm, address);
  352. spin_unlock(&kvm->mmu_lock);
  353. srcu_read_unlock(&kvm->srcu, idx);
  354. return young;
  355. }
  356. static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
  357. struct mm_struct *mm)
  358. {
  359. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  360. int idx;
  361. idx = srcu_read_lock(&kvm->srcu);
  362. kvm_arch_flush_shadow_all(kvm);
  363. srcu_read_unlock(&kvm->srcu, idx);
  364. }
  365. static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
  366. .invalidate_page = kvm_mmu_notifier_invalidate_page,
  367. .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
  368. .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
  369. .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
  370. .test_young = kvm_mmu_notifier_test_young,
  371. .change_pte = kvm_mmu_notifier_change_pte,
  372. .release = kvm_mmu_notifier_release,
  373. };
  374. static int kvm_init_mmu_notifier(struct kvm *kvm)
  375. {
  376. kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
  377. return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
  378. }
  379. #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
  380. static int kvm_init_mmu_notifier(struct kvm *kvm)
  381. {
  382. return 0;
  383. }
  384. #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
  385. static void kvm_init_memslots_id(struct kvm *kvm)
  386. {
  387. int i;
  388. struct kvm_memslots *slots = kvm->memslots;
  389. for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
  390. slots->id_to_index[i] = slots->memslots[i].id = i;
  391. }
  392. static struct kvm *kvm_create_vm(unsigned long type)
  393. {
  394. int r, i;
  395. struct kvm *kvm = kvm_arch_alloc_vm();
  396. if (!kvm)
  397. return ERR_PTR(-ENOMEM);
  398. r = kvm_arch_init_vm(kvm, type);
  399. if (r)
  400. goto out_err_nodisable;
  401. r = hardware_enable_all();
  402. if (r)
  403. goto out_err_nodisable;
  404. #ifdef CONFIG_HAVE_KVM_IRQCHIP
  405. INIT_HLIST_HEAD(&kvm->mask_notifier_list);
  406. INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
  407. #endif
  408. BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
  409. r = -ENOMEM;
  410. kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
  411. if (!kvm->memslots)
  412. goto out_err_nosrcu;
  413. kvm_init_memslots_id(kvm);
  414. if (init_srcu_struct(&kvm->srcu))
  415. goto out_err_nosrcu;
  416. for (i = 0; i < KVM_NR_BUSES; i++) {
  417. kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
  418. GFP_KERNEL);
  419. if (!kvm->buses[i])
  420. goto out_err;
  421. }
  422. spin_lock_init(&kvm->mmu_lock);
  423. kvm->mm = current->mm;
  424. atomic_inc(&kvm->mm->mm_count);
  425. kvm_eventfd_init(kvm);
  426. mutex_init(&kvm->lock);
  427. mutex_init(&kvm->irq_lock);
  428. mutex_init(&kvm->slots_lock);
  429. atomic_set(&kvm->users_count, 1);
  430. INIT_LIST_HEAD(&kvm->devices);
  431. r = kvm_init_mmu_notifier(kvm);
  432. if (r)
  433. goto out_err;
  434. raw_spin_lock(&kvm_lock);
  435. list_add(&kvm->vm_list, &vm_list);
  436. raw_spin_unlock(&kvm_lock);
  437. return kvm;
  438. out_err:
  439. cleanup_srcu_struct(&kvm->srcu);
  440. out_err_nosrcu:
  441. hardware_disable_all();
  442. out_err_nodisable:
  443. for (i = 0; i < KVM_NR_BUSES; i++)
  444. kfree(kvm->buses[i]);
  445. kfree(kvm->memslots);
  446. kvm_arch_free_vm(kvm);
  447. return ERR_PTR(r);
  448. }
  449. /*
  450. * Avoid using vmalloc for a small buffer.
  451. * Should not be used when the size is statically known.
  452. */
  453. void *kvm_kvzalloc(unsigned long size)
  454. {
  455. if (size > PAGE_SIZE)
  456. return vzalloc(size);
  457. else
  458. return kzalloc(size, GFP_KERNEL);
  459. }
  460. void kvm_kvfree(const void *addr)
  461. {
  462. if (is_vmalloc_addr(addr))
  463. vfree(addr);
  464. else
  465. kfree(addr);
  466. }
  467. static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
  468. {
  469. if (!memslot->dirty_bitmap)
  470. return;
  471. kvm_kvfree(memslot->dirty_bitmap);
  472. memslot->dirty_bitmap = NULL;
  473. }
  474. /*
  475. * Free any memory in @free but not in @dont.
  476. */
  477. static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
  478. struct kvm_memory_slot *dont)
  479. {
  480. if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
  481. kvm_destroy_dirty_bitmap(free);
  482. kvm_arch_free_memslot(free, dont);
  483. free->npages = 0;
  484. }
  485. void kvm_free_physmem(struct kvm *kvm)
  486. {
  487. struct kvm_memslots *slots = kvm->memslots;
  488. struct kvm_memory_slot *memslot;
  489. kvm_for_each_memslot(memslot, slots)
  490. kvm_free_physmem_slot(memslot, NULL);
  491. kfree(kvm->memslots);
  492. }
  493. static void kvm_destroy_devices(struct kvm *kvm)
  494. {
  495. struct list_head *node, *tmp;
  496. list_for_each_safe(node, tmp, &kvm->devices) {
  497. struct kvm_device *dev =
  498. list_entry(node, struct kvm_device, vm_node);
  499. list_del(node);
  500. dev->ops->destroy(dev);
  501. }
  502. }
  503. static void kvm_destroy_vm(struct kvm *kvm)
  504. {
  505. int i;
  506. struct mm_struct *mm = kvm->mm;
  507. kvm_arch_sync_events(kvm);
  508. raw_spin_lock(&kvm_lock);
  509. list_del(&kvm->vm_list);
  510. raw_spin_unlock(&kvm_lock);
  511. kvm_free_irq_routing(kvm);
  512. for (i = 0; i < KVM_NR_BUSES; i++)
  513. kvm_io_bus_destroy(kvm->buses[i]);
  514. kvm_coalesced_mmio_free(kvm);
  515. #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
  516. mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
  517. #else
  518. kvm_arch_flush_shadow_all(kvm);
  519. #endif
  520. kvm_arch_destroy_vm(kvm);
  521. kvm_destroy_devices(kvm);
  522. kvm_free_physmem(kvm);
  523. cleanup_srcu_struct(&kvm->srcu);
  524. kvm_arch_free_vm(kvm);
  525. hardware_disable_all();
  526. mmdrop(mm);
  527. }
  528. void kvm_get_kvm(struct kvm *kvm)
  529. {
  530. atomic_inc(&kvm->users_count);
  531. }
  532. EXPORT_SYMBOL_GPL(kvm_get_kvm);
  533. void kvm_put_kvm(struct kvm *kvm)
  534. {
  535. if (atomic_dec_and_test(&kvm->users_count))
  536. kvm_destroy_vm(kvm);
  537. }
  538. EXPORT_SYMBOL_GPL(kvm_put_kvm);
  539. static int kvm_vm_release(struct inode *inode, struct file *filp)
  540. {
  541. struct kvm *kvm = filp->private_data;
  542. kvm_irqfd_release(kvm);
  543. kvm_put_kvm(kvm);
  544. return 0;
  545. }
  546. /*
  547. * Allocation size is twice as large as the actual dirty bitmap size.
  548. * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
  549. */
  550. static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
  551. {
  552. #ifndef CONFIG_S390
  553. unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
  554. memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
  555. if (!memslot->dirty_bitmap)
  556. return -ENOMEM;
  557. #endif /* !CONFIG_S390 */
  558. return 0;
  559. }
  560. static int cmp_memslot(const void *slot1, const void *slot2)
  561. {
  562. struct kvm_memory_slot *s1, *s2;
  563. s1 = (struct kvm_memory_slot *)slot1;
  564. s2 = (struct kvm_memory_slot *)slot2;
  565. if (s1->npages < s2->npages)
  566. return 1;
  567. if (s1->npages > s2->npages)
  568. return -1;
  569. return 0;
  570. }
  571. /*
  572. * Sort the memslots base on its size, so the larger slots
  573. * will get better fit.
  574. */
  575. static void sort_memslots(struct kvm_memslots *slots)
  576. {
  577. int i;
  578. sort(slots->memslots, KVM_MEM_SLOTS_NUM,
  579. sizeof(struct kvm_memory_slot), cmp_memslot, NULL);
  580. for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
  581. slots->id_to_index[slots->memslots[i].id] = i;
  582. }
  583. void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new,
  584. u64 last_generation)
  585. {
  586. if (new) {
  587. int id = new->id;
  588. struct kvm_memory_slot *old = id_to_memslot(slots, id);
  589. unsigned long npages = old->npages;
  590. *old = *new;
  591. if (new->npages != npages)
  592. sort_memslots(slots);
  593. }
  594. slots->generation = last_generation + 1;
  595. }
  596. static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
  597. {
  598. u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
  599. #ifdef KVM_CAP_READONLY_MEM
  600. valid_flags |= KVM_MEM_READONLY;
  601. #endif
  602. if (mem->flags & ~valid_flags)
  603. return -EINVAL;
  604. return 0;
  605. }
  606. static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
  607. struct kvm_memslots *slots, struct kvm_memory_slot *new)
  608. {
  609. struct kvm_memslots *old_memslots = kvm->memslots;
  610. update_memslots(slots, new, kvm->memslots->generation);
  611. rcu_assign_pointer(kvm->memslots, slots);
  612. synchronize_srcu_expedited(&kvm->srcu);
  613. return old_memslots;
  614. }
  615. /*
  616. * Allocate some memory and give it an address in the guest physical address
  617. * space.
  618. *
  619. * Discontiguous memory is allowed, mostly for framebuffers.
  620. *
  621. * Must be called holding mmap_sem for write.
  622. */
  623. int __kvm_set_memory_region(struct kvm *kvm,
  624. struct kvm_userspace_memory_region *mem)
  625. {
  626. int r;
  627. gfn_t base_gfn;
  628. unsigned long npages;
  629. struct kvm_memory_slot *slot;
  630. struct kvm_memory_slot old, new;
  631. struct kvm_memslots *slots = NULL, *old_memslots;
  632. enum kvm_mr_change change;
  633. r = check_memory_region_flags(mem);
  634. if (r)
  635. goto out;
  636. r = -EINVAL;
  637. /* General sanity checks */
  638. if (mem->memory_size & (PAGE_SIZE - 1))
  639. goto out;
  640. if (mem->guest_phys_addr & (PAGE_SIZE - 1))
  641. goto out;
  642. /* We can read the guest memory with __xxx_user() later on. */
  643. if ((mem->slot < KVM_USER_MEM_SLOTS) &&
  644. ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
  645. !access_ok(VERIFY_WRITE,
  646. (void __user *)(unsigned long)mem->userspace_addr,
  647. mem->memory_size)))
  648. goto out;
  649. if (mem->slot >= KVM_MEM_SLOTS_NUM)
  650. goto out;
  651. if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
  652. goto out;
  653. slot = id_to_memslot(kvm->memslots, mem->slot);
  654. base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
  655. npages = mem->memory_size >> PAGE_SHIFT;
  656. r = -EINVAL;
  657. if (npages > KVM_MEM_MAX_NR_PAGES)
  658. goto out;
  659. if (!npages)
  660. mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
  661. new = old = *slot;
  662. new.id = mem->slot;
  663. new.base_gfn = base_gfn;
  664. new.npages = npages;
  665. new.flags = mem->flags;
  666. r = -EINVAL;
  667. if (npages) {
  668. if (!old.npages)
  669. change = KVM_MR_CREATE;
  670. else { /* Modify an existing slot. */
  671. if ((mem->userspace_addr != old.userspace_addr) ||
  672. (npages != old.npages) ||
  673. ((new.flags ^ old.flags) & KVM_MEM_READONLY))
  674. goto out;
  675. if (base_gfn != old.base_gfn)
  676. change = KVM_MR_MOVE;
  677. else if (new.flags != old.flags)
  678. change = KVM_MR_FLAGS_ONLY;
  679. else { /* Nothing to change. */
  680. r = 0;
  681. goto out;
  682. }
  683. }
  684. } else if (old.npages) {
  685. change = KVM_MR_DELETE;
  686. } else /* Modify a non-existent slot: disallowed. */
  687. goto out;
  688. if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
  689. /* Check for overlaps */
  690. r = -EEXIST;
  691. kvm_for_each_memslot(slot, kvm->memslots) {
  692. if ((slot->id >= KVM_USER_MEM_SLOTS) ||
  693. (slot->id == mem->slot))
  694. continue;
  695. if (!((base_gfn + npages <= slot->base_gfn) ||
  696. (base_gfn >= slot->base_gfn + slot->npages)))
  697. goto out;
  698. }
  699. }
  700. /* Free page dirty bitmap if unneeded */
  701. if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
  702. new.dirty_bitmap = NULL;
  703. r = -ENOMEM;
  704. if (change == KVM_MR_CREATE) {
  705. new.userspace_addr = mem->userspace_addr;
  706. if (kvm_arch_create_memslot(&new, npages))
  707. goto out_free;
  708. }
  709. /* Allocate page dirty bitmap if needed */
  710. if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
  711. if (kvm_create_dirty_bitmap(&new) < 0)
  712. goto out_free;
  713. }
  714. if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
  715. r = -ENOMEM;
  716. slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
  717. GFP_KERNEL);
  718. if (!slots)
  719. goto out_free;
  720. slot = id_to_memslot(slots, mem->slot);
  721. slot->flags |= KVM_MEMSLOT_INVALID;
  722. old_memslots = install_new_memslots(kvm, slots, NULL);
  723. /* slot was deleted or moved, clear iommu mapping */
  724. kvm_iommu_unmap_pages(kvm, &old);
  725. /* From this point no new shadow pages pointing to a deleted,
  726. * or moved, memslot will be created.
  727. *
  728. * validation of sp->gfn happens in:
  729. * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
  730. * - kvm_is_visible_gfn (mmu_check_roots)
  731. */
  732. kvm_arch_flush_shadow_memslot(kvm, slot);
  733. slots = old_memslots;
  734. }
  735. r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
  736. if (r)
  737. goto out_slots;
  738. r = -ENOMEM;
  739. /*
  740. * We can re-use the old_memslots from above, the only difference
  741. * from the currently installed memslots is the invalid flag. This
  742. * will get overwritten by update_memslots anyway.
  743. */
  744. if (!slots) {
  745. slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
  746. GFP_KERNEL);
  747. if (!slots)
  748. goto out_free;
  749. }
  750. /*
  751. * IOMMU mapping: New slots need to be mapped. Old slots need to be
  752. * un-mapped and re-mapped if their base changes. Since base change
  753. * unmapping is handled above with slot deletion, mapping alone is
  754. * needed here. Anything else the iommu might care about for existing
  755. * slots (size changes, userspace addr changes and read-only flag
  756. * changes) is disallowed above, so any other attribute changes getting
  757. * here can be skipped.
  758. */
  759. if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
  760. r = kvm_iommu_map_pages(kvm, &new);
  761. if (r)
  762. goto out_slots;
  763. }
  764. /* actual memory is freed via old in kvm_free_physmem_slot below */
  765. if (change == KVM_MR_DELETE) {
  766. new.dirty_bitmap = NULL;
  767. memset(&new.arch, 0, sizeof(new.arch));
  768. }
  769. old_memslots = install_new_memslots(kvm, slots, &new);
  770. kvm_arch_commit_memory_region(kvm, mem, &old, change);
  771. kvm_free_physmem_slot(&old, &new);
  772. kfree(old_memslots);
  773. return 0;
  774. out_slots:
  775. kfree(slots);
  776. out_free:
  777. kvm_free_physmem_slot(&new, &old);
  778. out:
  779. return r;
  780. }
  781. EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
  782. int kvm_set_memory_region(struct kvm *kvm,
  783. struct kvm_userspace_memory_region *mem)
  784. {
  785. int r;
  786. mutex_lock(&kvm->slots_lock);
  787. r = __kvm_set_memory_region(kvm, mem);
  788. mutex_unlock(&kvm->slots_lock);
  789. return r;
  790. }
  791. EXPORT_SYMBOL_GPL(kvm_set_memory_region);
  792. int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  793. struct kvm_userspace_memory_region *mem)
  794. {
  795. if (mem->slot >= KVM_USER_MEM_SLOTS)
  796. return -EINVAL;
  797. return kvm_set_memory_region(kvm, mem);
  798. }
  799. int kvm_get_dirty_log(struct kvm *kvm,
  800. struct kvm_dirty_log *log, int *is_dirty)
  801. {
  802. struct kvm_memory_slot *memslot;
  803. int r, i;
  804. unsigned long n;
  805. unsigned long any = 0;
  806. r = -EINVAL;
  807. if (log->slot >= KVM_USER_MEM_SLOTS)
  808. goto out;
  809. memslot = id_to_memslot(kvm->memslots, log->slot);
  810. r = -ENOENT;
  811. if (!memslot->dirty_bitmap)
  812. goto out;
  813. n = kvm_dirty_bitmap_bytes(memslot);
  814. for (i = 0; !any && i < n/sizeof(long); ++i)
  815. any = memslot->dirty_bitmap[i];
  816. r = -EFAULT;
  817. if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
  818. goto out;
  819. if (any)
  820. *is_dirty = 1;
  821. r = 0;
  822. out:
  823. return r;
  824. }
  825. bool kvm_largepages_enabled(void)
  826. {
  827. return largepages_enabled;
  828. }
  829. void kvm_disable_largepages(void)
  830. {
  831. largepages_enabled = false;
  832. }
  833. EXPORT_SYMBOL_GPL(kvm_disable_largepages);
  834. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  835. {
  836. return __gfn_to_memslot(kvm_memslots(kvm), gfn);
  837. }
  838. EXPORT_SYMBOL_GPL(gfn_to_memslot);
  839. int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
  840. {
  841. struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
  842. if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
  843. memslot->flags & KVM_MEMSLOT_INVALID)
  844. return 0;
  845. return 1;
  846. }
  847. EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
  848. unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
  849. {
  850. struct vm_area_struct *vma;
  851. unsigned long addr, size;
  852. size = PAGE_SIZE;
  853. addr = gfn_to_hva(kvm, gfn);
  854. if (kvm_is_error_hva(addr))
  855. return PAGE_SIZE;
  856. down_read(&current->mm->mmap_sem);
  857. vma = find_vma(current->mm, addr);
  858. if (!vma)
  859. goto out;
  860. size = vma_kernel_pagesize(vma);
  861. out:
  862. up_read(&current->mm->mmap_sem);
  863. return size;
  864. }
  865. static bool memslot_is_readonly(struct kvm_memory_slot *slot)
  866. {
  867. return slot->flags & KVM_MEM_READONLY;
  868. }
  869. static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
  870. gfn_t *nr_pages, bool write)
  871. {
  872. if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
  873. return KVM_HVA_ERR_BAD;
  874. if (memslot_is_readonly(slot) && write)
  875. return KVM_HVA_ERR_RO_BAD;
  876. if (nr_pages)
  877. *nr_pages = slot->npages - (gfn - slot->base_gfn);
  878. return __gfn_to_hva_memslot(slot, gfn);
  879. }
  880. static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
  881. gfn_t *nr_pages)
  882. {
  883. return __gfn_to_hva_many(slot, gfn, nr_pages, true);
  884. }
  885. unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
  886. gfn_t gfn)
  887. {
  888. return gfn_to_hva_many(slot, gfn, NULL);
  889. }
  890. EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
  891. unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
  892. {
  893. return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
  894. }
  895. EXPORT_SYMBOL_GPL(gfn_to_hva);
  896. /*
  897. * The hva returned by this function is only allowed to be read.
  898. * It should pair with kvm_read_hva() or kvm_read_hva_atomic().
  899. */
  900. static unsigned long gfn_to_hva_read(struct kvm *kvm, gfn_t gfn)
  901. {
  902. return __gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL, false);
  903. }
  904. static int kvm_read_hva(void *data, void __user *hva, int len)
  905. {
  906. return __copy_from_user(data, hva, len);
  907. }
  908. static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
  909. {
  910. return __copy_from_user_inatomic(data, hva, len);
  911. }
  912. static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
  913. unsigned long start, int write, struct page **page)
  914. {
  915. int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
  916. if (write)
  917. flags |= FOLL_WRITE;
  918. return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
  919. }
  920. static inline int check_user_page_hwpoison(unsigned long addr)
  921. {
  922. int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
  923. rc = __get_user_pages(current, current->mm, addr, 1,
  924. flags, NULL, NULL, NULL);
  925. return rc == -EHWPOISON;
  926. }
  927. /*
  928. * The atomic path to get the writable pfn which will be stored in @pfn,
  929. * true indicates success, otherwise false is returned.
  930. */
  931. static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
  932. bool write_fault, bool *writable, pfn_t *pfn)
  933. {
  934. struct page *page[1];
  935. int npages;
  936. if (!(async || atomic))
  937. return false;
  938. /*
  939. * Fast pin a writable pfn only if it is a write fault request
  940. * or the caller allows to map a writable pfn for a read fault
  941. * request.
  942. */
  943. if (!(write_fault || writable))
  944. return false;
  945. npages = __get_user_pages_fast(addr, 1, 1, page);
  946. if (npages == 1) {
  947. *pfn = page_to_pfn(page[0]);
  948. if (writable)
  949. *writable = true;
  950. return true;
  951. }
  952. return false;
  953. }
  954. /*
  955. * The slow path to get the pfn of the specified host virtual address,
  956. * 1 indicates success, -errno is returned if error is detected.
  957. */
  958. static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
  959. bool *writable, pfn_t *pfn)
  960. {
  961. struct page *page[1];
  962. int npages = 0;
  963. might_sleep();
  964. if (writable)
  965. *writable = write_fault;
  966. if (async) {
  967. down_read(&current->mm->mmap_sem);
  968. npages = get_user_page_nowait(current, current->mm,
  969. addr, write_fault, page);
  970. up_read(&current->mm->mmap_sem);
  971. } else
  972. npages = get_user_pages_fast(addr, 1, write_fault,
  973. page);
  974. if (npages != 1)
  975. return npages;
  976. /* map read fault as writable if possible */
  977. if (unlikely(!write_fault) && writable) {
  978. struct page *wpage[1];
  979. npages = __get_user_pages_fast(addr, 1, 1, wpage);
  980. if (npages == 1) {
  981. *writable = true;
  982. put_page(page[0]);
  983. page[0] = wpage[0];
  984. }
  985. npages = 1;
  986. }
  987. *pfn = page_to_pfn(page[0]);
  988. return npages;
  989. }
  990. static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
  991. {
  992. if (unlikely(!(vma->vm_flags & VM_READ)))
  993. return false;
  994. if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
  995. return false;
  996. return true;
  997. }
  998. /*
  999. * Pin guest page in memory and return its pfn.
  1000. * @addr: host virtual address which maps memory to the guest
  1001. * @atomic: whether this function can sleep
  1002. * @async: whether this function need to wait IO complete if the
  1003. * host page is not in the memory
  1004. * @write_fault: whether we should get a writable host page
  1005. * @writable: whether it allows to map a writable host page for !@write_fault
  1006. *
  1007. * The function will map a writable host page for these two cases:
  1008. * 1): @write_fault = true
  1009. * 2): @write_fault = false && @writable, @writable will tell the caller
  1010. * whether the mapping is writable.
  1011. */
  1012. static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
  1013. bool write_fault, bool *writable)
  1014. {
  1015. struct vm_area_struct *vma;
  1016. pfn_t pfn = 0;
  1017. int npages;
  1018. /* we can do it either atomically or asynchronously, not both */
  1019. BUG_ON(atomic && async);
  1020. if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
  1021. return pfn;
  1022. if (atomic)
  1023. return KVM_PFN_ERR_FAULT;
  1024. npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
  1025. if (npages == 1)
  1026. return pfn;
  1027. down_read(&current->mm->mmap_sem);
  1028. if (npages == -EHWPOISON ||
  1029. (!async && check_user_page_hwpoison(addr))) {
  1030. pfn = KVM_PFN_ERR_HWPOISON;
  1031. goto exit;
  1032. }
  1033. vma = find_vma_intersection(current->mm, addr, addr + 1);
  1034. if (vma == NULL)
  1035. pfn = KVM_PFN_ERR_FAULT;
  1036. else if ((vma->vm_flags & VM_PFNMAP)) {
  1037. pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
  1038. vma->vm_pgoff;
  1039. BUG_ON(!kvm_is_mmio_pfn(pfn));
  1040. } else {
  1041. if (async && vma_is_valid(vma, write_fault))
  1042. *async = true;
  1043. pfn = KVM_PFN_ERR_FAULT;
  1044. }
  1045. exit:
  1046. up_read(&current->mm->mmap_sem);
  1047. return pfn;
  1048. }
  1049. static pfn_t
  1050. __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
  1051. bool *async, bool write_fault, bool *writable)
  1052. {
  1053. unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
  1054. if (addr == KVM_HVA_ERR_RO_BAD)
  1055. return KVM_PFN_ERR_RO_FAULT;
  1056. if (kvm_is_error_hva(addr))
  1057. return KVM_PFN_NOSLOT;
  1058. /* Do not map writable pfn in the readonly memslot. */
  1059. if (writable && memslot_is_readonly(slot)) {
  1060. *writable = false;
  1061. writable = NULL;
  1062. }
  1063. return hva_to_pfn(addr, atomic, async, write_fault,
  1064. writable);
  1065. }
  1066. static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
  1067. bool write_fault, bool *writable)
  1068. {
  1069. struct kvm_memory_slot *slot;
  1070. if (async)
  1071. *async = false;
  1072. slot = gfn_to_memslot(kvm, gfn);
  1073. return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
  1074. writable);
  1075. }
  1076. pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
  1077. {
  1078. return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
  1079. }
  1080. EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
  1081. pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
  1082. bool write_fault, bool *writable)
  1083. {
  1084. return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
  1085. }
  1086. EXPORT_SYMBOL_GPL(gfn_to_pfn_async);
  1087. pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
  1088. {
  1089. return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
  1090. }
  1091. EXPORT_SYMBOL_GPL(gfn_to_pfn);
  1092. pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
  1093. bool *writable)
  1094. {
  1095. return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
  1096. }
  1097. EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
  1098. pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
  1099. {
  1100. return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
  1101. }
  1102. pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
  1103. {
  1104. return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
  1105. }
  1106. EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
  1107. int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
  1108. int nr_pages)
  1109. {
  1110. unsigned long addr;
  1111. gfn_t entry;
  1112. addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
  1113. if (kvm_is_error_hva(addr))
  1114. return -1;
  1115. if (entry < nr_pages)
  1116. return 0;
  1117. return __get_user_pages_fast(addr, nr_pages, 1, pages);
  1118. }
  1119. EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
  1120. static struct page *kvm_pfn_to_page(pfn_t pfn)
  1121. {
  1122. if (is_error_noslot_pfn(pfn))
  1123. return KVM_ERR_PTR_BAD_PAGE;
  1124. if (kvm_is_mmio_pfn(pfn)) {
  1125. WARN_ON(1);
  1126. return KVM_ERR_PTR_BAD_PAGE;
  1127. }
  1128. return pfn_to_page(pfn);
  1129. }
  1130. struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
  1131. {
  1132. pfn_t pfn;
  1133. pfn = gfn_to_pfn(kvm, gfn);
  1134. return kvm_pfn_to_page(pfn);
  1135. }
  1136. EXPORT_SYMBOL_GPL(gfn_to_page);
  1137. void kvm_release_page_clean(struct page *page)
  1138. {
  1139. WARN_ON(is_error_page(page));
  1140. kvm_release_pfn_clean(page_to_pfn(page));
  1141. }
  1142. EXPORT_SYMBOL_GPL(kvm_release_page_clean);
  1143. void kvm_release_pfn_clean(pfn_t pfn)
  1144. {
  1145. if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn))
  1146. put_page(pfn_to_page(pfn));
  1147. }
  1148. EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
  1149. void kvm_release_page_dirty(struct page *page)
  1150. {
  1151. WARN_ON(is_error_page(page));
  1152. kvm_release_pfn_dirty(page_to_pfn(page));
  1153. }
  1154. EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
  1155. void kvm_release_pfn_dirty(pfn_t pfn)
  1156. {
  1157. kvm_set_pfn_dirty(pfn);
  1158. kvm_release_pfn_clean(pfn);
  1159. }
  1160. EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
  1161. void kvm_set_page_dirty(struct page *page)
  1162. {
  1163. kvm_set_pfn_dirty(page_to_pfn(page));
  1164. }
  1165. EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
  1166. void kvm_set_pfn_dirty(pfn_t pfn)
  1167. {
  1168. if (!kvm_is_mmio_pfn(pfn)) {
  1169. struct page *page = pfn_to_page(pfn);
  1170. if (!PageReserved(page))
  1171. SetPageDirty(page);
  1172. }
  1173. }
  1174. EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
  1175. void kvm_set_pfn_accessed(pfn_t pfn)
  1176. {
  1177. if (!kvm_is_mmio_pfn(pfn))
  1178. mark_page_accessed(pfn_to_page(pfn));
  1179. }
  1180. EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
  1181. void kvm_get_pfn(pfn_t pfn)
  1182. {
  1183. if (!kvm_is_mmio_pfn(pfn))
  1184. get_page(pfn_to_page(pfn));
  1185. }
  1186. EXPORT_SYMBOL_GPL(kvm_get_pfn);
  1187. static int next_segment(unsigned long len, int offset)
  1188. {
  1189. if (len > PAGE_SIZE - offset)
  1190. return PAGE_SIZE - offset;
  1191. else
  1192. return len;
  1193. }
  1194. int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
  1195. int len)
  1196. {
  1197. int r;
  1198. unsigned long addr;
  1199. addr = gfn_to_hva_read(kvm, gfn);
  1200. if (kvm_is_error_hva(addr))
  1201. return -EFAULT;
  1202. r = kvm_read_hva(data, (void __user *)addr + offset, len);
  1203. if (r)
  1204. return -EFAULT;
  1205. return 0;
  1206. }
  1207. EXPORT_SYMBOL_GPL(kvm_read_guest_page);
  1208. int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
  1209. {
  1210. gfn_t gfn = gpa >> PAGE_SHIFT;
  1211. int seg;
  1212. int offset = offset_in_page(gpa);
  1213. int ret;
  1214. while ((seg = next_segment(len, offset)) != 0) {
  1215. ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
  1216. if (ret < 0)
  1217. return ret;
  1218. offset = 0;
  1219. len -= seg;
  1220. data += seg;
  1221. ++gfn;
  1222. }
  1223. return 0;
  1224. }
  1225. EXPORT_SYMBOL_GPL(kvm_read_guest);
  1226. int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
  1227. unsigned long len)
  1228. {
  1229. int r;
  1230. unsigned long addr;
  1231. gfn_t gfn = gpa >> PAGE_SHIFT;
  1232. int offset = offset_in_page(gpa);
  1233. addr = gfn_to_hva_read(kvm, gfn);
  1234. if (kvm_is_error_hva(addr))
  1235. return -EFAULT;
  1236. pagefault_disable();
  1237. r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
  1238. pagefault_enable();
  1239. if (r)
  1240. return -EFAULT;
  1241. return 0;
  1242. }
  1243. EXPORT_SYMBOL(kvm_read_guest_atomic);
  1244. int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
  1245. int offset, int len)
  1246. {
  1247. int r;
  1248. unsigned long addr;
  1249. addr = gfn_to_hva(kvm, gfn);
  1250. if (kvm_is_error_hva(addr))
  1251. return -EFAULT;
  1252. r = __copy_to_user((void __user *)addr + offset, data, len);
  1253. if (r)
  1254. return -EFAULT;
  1255. mark_page_dirty(kvm, gfn);
  1256. return 0;
  1257. }
  1258. EXPORT_SYMBOL_GPL(kvm_write_guest_page);
  1259. int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
  1260. unsigned long len)
  1261. {
  1262. gfn_t gfn = gpa >> PAGE_SHIFT;
  1263. int seg;
  1264. int offset = offset_in_page(gpa);
  1265. int ret;
  1266. while ((seg = next_segment(len, offset)) != 0) {
  1267. ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
  1268. if (ret < 0)
  1269. return ret;
  1270. offset = 0;
  1271. len -= seg;
  1272. data += seg;
  1273. ++gfn;
  1274. }
  1275. return 0;
  1276. }
  1277. int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1278. gpa_t gpa, unsigned long len)
  1279. {
  1280. struct kvm_memslots *slots = kvm_memslots(kvm);
  1281. int offset = offset_in_page(gpa);
  1282. gfn_t start_gfn = gpa >> PAGE_SHIFT;
  1283. gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
  1284. gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
  1285. gfn_t nr_pages_avail;
  1286. ghc->gpa = gpa;
  1287. ghc->generation = slots->generation;
  1288. ghc->len = len;
  1289. ghc->memslot = gfn_to_memslot(kvm, start_gfn);
  1290. ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, &nr_pages_avail);
  1291. if (!kvm_is_error_hva(ghc->hva) && nr_pages_avail >= nr_pages_needed) {
  1292. ghc->hva += offset;
  1293. } else {
  1294. /*
  1295. * If the requested region crosses two memslots, we still
  1296. * verify that the entire region is valid here.
  1297. */
  1298. while (start_gfn <= end_gfn) {
  1299. ghc->memslot = gfn_to_memslot(kvm, start_gfn);
  1300. ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
  1301. &nr_pages_avail);
  1302. if (kvm_is_error_hva(ghc->hva))
  1303. return -EFAULT;
  1304. start_gfn += nr_pages_avail;
  1305. }
  1306. /* Use the slow path for cross page reads and writes. */
  1307. ghc->memslot = NULL;
  1308. }
  1309. return 0;
  1310. }
  1311. EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
  1312. int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1313. void *data, unsigned long len)
  1314. {
  1315. struct kvm_memslots *slots = kvm_memslots(kvm);
  1316. int r;
  1317. BUG_ON(len > ghc->len);
  1318. if (slots->generation != ghc->generation)
  1319. kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
  1320. if (unlikely(!ghc->memslot))
  1321. return kvm_write_guest(kvm, ghc->gpa, data, len);
  1322. if (kvm_is_error_hva(ghc->hva))
  1323. return -EFAULT;
  1324. r = __copy_to_user((void __user *)ghc->hva, data, len);
  1325. if (r)
  1326. return -EFAULT;
  1327. mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);
  1328. return 0;
  1329. }
  1330. EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
  1331. int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1332. void *data, unsigned long len)
  1333. {
  1334. struct kvm_memslots *slots = kvm_memslots(kvm);
  1335. int r;
  1336. BUG_ON(len > ghc->len);
  1337. if (slots->generation != ghc->generation)
  1338. kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
  1339. if (unlikely(!ghc->memslot))
  1340. return kvm_read_guest(kvm, ghc->gpa, data, len);
  1341. if (kvm_is_error_hva(ghc->hva))
  1342. return -EFAULT;
  1343. r = __copy_from_user(data, (void __user *)ghc->hva, len);
  1344. if (r)
  1345. return -EFAULT;
  1346. return 0;
  1347. }
  1348. EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
  1349. int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
  1350. {
  1351. return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
  1352. offset, len);
  1353. }
  1354. EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
  1355. int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
  1356. {
  1357. gfn_t gfn = gpa >> PAGE_SHIFT;
  1358. int seg;
  1359. int offset = offset_in_page(gpa);
  1360. int ret;
  1361. while ((seg = next_segment(len, offset)) != 0) {
  1362. ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
  1363. if (ret < 0)
  1364. return ret;
  1365. offset = 0;
  1366. len -= seg;
  1367. ++gfn;
  1368. }
  1369. return 0;
  1370. }
  1371. EXPORT_SYMBOL_GPL(kvm_clear_guest);
  1372. void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
  1373. gfn_t gfn)
  1374. {
  1375. if (memslot && memslot->dirty_bitmap) {
  1376. unsigned long rel_gfn = gfn - memslot->base_gfn;
  1377. set_bit_le(rel_gfn, memslot->dirty_bitmap);
  1378. }
  1379. }
  1380. void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
  1381. {
  1382. struct kvm_memory_slot *memslot;
  1383. memslot = gfn_to_memslot(kvm, gfn);
  1384. mark_page_dirty_in_slot(kvm, memslot, gfn);
  1385. }
  1386. /*
  1387. * The vCPU has executed a HLT instruction with in-kernel mode enabled.
  1388. */
  1389. void kvm_vcpu_block(struct kvm_vcpu *vcpu)
  1390. {
  1391. DEFINE_WAIT(wait);
  1392. for (;;) {
  1393. prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
  1394. if (kvm_arch_vcpu_runnable(vcpu)) {
  1395. kvm_make_request(KVM_REQ_UNHALT, vcpu);
  1396. break;
  1397. }
  1398. if (kvm_cpu_has_pending_timer(vcpu))
  1399. break;
  1400. if (signal_pending(current))
  1401. break;
  1402. schedule();
  1403. }
  1404. finish_wait(&vcpu->wq, &wait);
  1405. }
  1406. #ifndef CONFIG_S390
  1407. /*
  1408. * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
  1409. */
  1410. void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
  1411. {
  1412. int me;
  1413. int cpu = vcpu->cpu;
  1414. wait_queue_head_t *wqp;
  1415. wqp = kvm_arch_vcpu_wq(vcpu);
  1416. if (waitqueue_active(wqp)) {
  1417. wake_up_interruptible(wqp);
  1418. ++vcpu->stat.halt_wakeup;
  1419. }
  1420. me = get_cpu();
  1421. if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
  1422. if (kvm_arch_vcpu_should_kick(vcpu))
  1423. smp_send_reschedule(cpu);
  1424. put_cpu();
  1425. }
  1426. EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
  1427. #endif /* !CONFIG_S390 */
  1428. void kvm_resched(struct kvm_vcpu *vcpu)
  1429. {
  1430. if (!need_resched())
  1431. return;
  1432. cond_resched();
  1433. }
  1434. EXPORT_SYMBOL_GPL(kvm_resched);
  1435. bool kvm_vcpu_yield_to(struct kvm_vcpu *target)
  1436. {
  1437. struct pid *pid;
  1438. struct task_struct *task = NULL;
  1439. bool ret = false;
  1440. rcu_read_lock();
  1441. pid = rcu_dereference(target->pid);
  1442. if (pid)
  1443. task = get_pid_task(target->pid, PIDTYPE_PID);
  1444. rcu_read_unlock();
  1445. if (!task)
  1446. return ret;
  1447. if (task->flags & PF_VCPU) {
  1448. put_task_struct(task);
  1449. return ret;
  1450. }
  1451. ret = yield_to(task, 1);
  1452. put_task_struct(task);
  1453. return ret;
  1454. }
  1455. EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
  1456. #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
  1457. /*
  1458. * Helper that checks whether a VCPU is eligible for directed yield.
  1459. * Most eligible candidate to yield is decided by following heuristics:
  1460. *
  1461. * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
  1462. * (preempted lock holder), indicated by @in_spin_loop.
  1463. * Set at the beiginning and cleared at the end of interception/PLE handler.
  1464. *
  1465. * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
  1466. * chance last time (mostly it has become eligible now since we have probably
  1467. * yielded to lockholder in last iteration. This is done by toggling
  1468. * @dy_eligible each time a VCPU checked for eligibility.)
  1469. *
  1470. * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
  1471. * to preempted lock-holder could result in wrong VCPU selection and CPU
  1472. * burning. Giving priority for a potential lock-holder increases lock
  1473. * progress.
  1474. *
  1475. * Since algorithm is based on heuristics, accessing another VCPU data without
  1476. * locking does not harm. It may result in trying to yield to same VCPU, fail
  1477. * and continue with next VCPU and so on.
  1478. */
  1479. bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
  1480. {
  1481. bool eligible;
  1482. eligible = !vcpu->spin_loop.in_spin_loop ||
  1483. (vcpu->spin_loop.in_spin_loop &&
  1484. vcpu->spin_loop.dy_eligible);
  1485. if (vcpu->spin_loop.in_spin_loop)
  1486. kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
  1487. return eligible;
  1488. }
  1489. #endif
  1490. void kvm_vcpu_on_spin(struct kvm_vcpu *me)
  1491. {
  1492. struct kvm *kvm = me->kvm;
  1493. struct kvm_vcpu *vcpu;
  1494. int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
  1495. int yielded = 0;
  1496. int try = 3;
  1497. int pass;
  1498. int i;
  1499. kvm_vcpu_set_in_spin_loop(me, true);
  1500. /*
  1501. * We boost the priority of a VCPU that is runnable but not
  1502. * currently running, because it got preempted by something
  1503. * else and called schedule in __vcpu_run. Hopefully that
  1504. * VCPU is holding the lock that we need and will release it.
  1505. * We approximate round-robin by starting at the last boosted VCPU.
  1506. */
  1507. for (pass = 0; pass < 2 && !yielded && try; pass++) {
  1508. kvm_for_each_vcpu(i, vcpu, kvm) {
  1509. if (!pass && i <= last_boosted_vcpu) {
  1510. i = last_boosted_vcpu;
  1511. continue;
  1512. } else if (pass && i > last_boosted_vcpu)
  1513. break;
  1514. if (!ACCESS_ONCE(vcpu->preempted))
  1515. continue;
  1516. if (vcpu == me)
  1517. continue;
  1518. if (waitqueue_active(&vcpu->wq))
  1519. continue;
  1520. if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
  1521. continue;
  1522. yielded = kvm_vcpu_yield_to(vcpu);
  1523. if (yielded > 0) {
  1524. kvm->last_boosted_vcpu = i;
  1525. break;
  1526. } else if (yielded < 0) {
  1527. try--;
  1528. if (!try)
  1529. break;
  1530. }
  1531. }
  1532. }
  1533. kvm_vcpu_set_in_spin_loop(me, false);
  1534. /* Ensure vcpu is not eligible during next spinloop */
  1535. kvm_vcpu_set_dy_eligible(me, false);
  1536. }
  1537. EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
  1538. static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1539. {
  1540. struct kvm_vcpu *vcpu = vma->vm_file->private_data;
  1541. struct page *page;
  1542. if (vmf->pgoff == 0)
  1543. page = virt_to_page(vcpu->run);
  1544. #ifdef CONFIG_X86
  1545. else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
  1546. page = virt_to_page(vcpu->arch.pio_data);
  1547. #endif
  1548. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  1549. else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
  1550. page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
  1551. #endif
  1552. else
  1553. return kvm_arch_vcpu_fault(vcpu, vmf);
  1554. get_page(page);
  1555. vmf->page = page;
  1556. return 0;
  1557. }
  1558. static const struct vm_operations_struct kvm_vcpu_vm_ops = {
  1559. .fault = kvm_vcpu_fault,
  1560. };
  1561. static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
  1562. {
  1563. vma->vm_ops = &kvm_vcpu_vm_ops;
  1564. return 0;
  1565. }
  1566. static int kvm_vcpu_release(struct inode *inode, struct file *filp)
  1567. {
  1568. struct kvm_vcpu *vcpu = filp->private_data;
  1569. kvm_put_kvm(vcpu->kvm);
  1570. return 0;
  1571. }
  1572. static struct file_operations kvm_vcpu_fops = {
  1573. .release = kvm_vcpu_release,
  1574. .unlocked_ioctl = kvm_vcpu_ioctl,
  1575. #ifdef CONFIG_COMPAT
  1576. .compat_ioctl = kvm_vcpu_compat_ioctl,
  1577. #endif
  1578. .mmap = kvm_vcpu_mmap,
  1579. .llseek = noop_llseek,
  1580. };
  1581. /*
  1582. * Allocates an inode for the vcpu.
  1583. */
  1584. static int create_vcpu_fd(struct kvm_vcpu *vcpu)
  1585. {
  1586. return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
  1587. }
  1588. /*
  1589. * Creates some virtual cpus. Good luck creating more than one.
  1590. */
  1591. static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
  1592. {
  1593. int r;
  1594. struct kvm_vcpu *vcpu, *v;
  1595. vcpu = kvm_arch_vcpu_create(kvm, id);
  1596. if (IS_ERR(vcpu))
  1597. return PTR_ERR(vcpu);
  1598. preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
  1599. r = kvm_arch_vcpu_setup(vcpu);
  1600. if (r)
  1601. goto vcpu_destroy;
  1602. mutex_lock(&kvm->lock);
  1603. if (!kvm_vcpu_compatible(vcpu)) {
  1604. r = -EINVAL;
  1605. goto unlock_vcpu_destroy;
  1606. }
  1607. if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
  1608. r = -EINVAL;
  1609. goto unlock_vcpu_destroy;
  1610. }
  1611. kvm_for_each_vcpu(r, v, kvm)
  1612. if (v->vcpu_id == id) {
  1613. r = -EEXIST;
  1614. goto unlock_vcpu_destroy;
  1615. }
  1616. BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
  1617. /* Now it's all set up, let userspace reach it */
  1618. kvm_get_kvm(kvm);
  1619. r = create_vcpu_fd(vcpu);
  1620. if (r < 0) {
  1621. kvm_put_kvm(kvm);
  1622. goto unlock_vcpu_destroy;
  1623. }
  1624. kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
  1625. smp_wmb();
  1626. atomic_inc(&kvm->online_vcpus);
  1627. mutex_unlock(&kvm->lock);
  1628. kvm_arch_vcpu_postcreate(vcpu);
  1629. return r;
  1630. unlock_vcpu_destroy:
  1631. mutex_unlock(&kvm->lock);
  1632. vcpu_destroy:
  1633. kvm_arch_vcpu_destroy(vcpu);
  1634. return r;
  1635. }
  1636. static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
  1637. {
  1638. if (sigset) {
  1639. sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
  1640. vcpu->sigset_active = 1;
  1641. vcpu->sigset = *sigset;
  1642. } else
  1643. vcpu->sigset_active = 0;
  1644. return 0;
  1645. }
  1646. static long kvm_vcpu_ioctl(struct file *filp,
  1647. unsigned int ioctl, unsigned long arg)
  1648. {
  1649. struct kvm_vcpu *vcpu = filp->private_data;
  1650. void __user *argp = (void __user *)arg;
  1651. int r;
  1652. struct kvm_fpu *fpu = NULL;
  1653. struct kvm_sregs *kvm_sregs = NULL;
  1654. if (vcpu->kvm->mm != current->mm)
  1655. return -EIO;
  1656. #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
  1657. /*
  1658. * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
  1659. * so vcpu_load() would break it.
  1660. */
  1661. if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
  1662. return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  1663. #endif
  1664. r = vcpu_load(vcpu);
  1665. if (r)
  1666. return r;
  1667. switch (ioctl) {
  1668. case KVM_RUN:
  1669. r = -EINVAL;
  1670. if (arg)
  1671. goto out;
  1672. r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
  1673. trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
  1674. break;
  1675. case KVM_GET_REGS: {
  1676. struct kvm_regs *kvm_regs;
  1677. r = -ENOMEM;
  1678. kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
  1679. if (!kvm_regs)
  1680. goto out;
  1681. r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
  1682. if (r)
  1683. goto out_free1;
  1684. r = -EFAULT;
  1685. if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
  1686. goto out_free1;
  1687. r = 0;
  1688. out_free1:
  1689. kfree(kvm_regs);
  1690. break;
  1691. }
  1692. case KVM_SET_REGS: {
  1693. struct kvm_regs *kvm_regs;
  1694. r = -ENOMEM;
  1695. kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
  1696. if (IS_ERR(kvm_regs)) {
  1697. r = PTR_ERR(kvm_regs);
  1698. goto out;
  1699. }
  1700. r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
  1701. kfree(kvm_regs);
  1702. break;
  1703. }
  1704. case KVM_GET_SREGS: {
  1705. kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
  1706. r = -ENOMEM;
  1707. if (!kvm_sregs)
  1708. goto out;
  1709. r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
  1710. if (r)
  1711. goto out;
  1712. r = -EFAULT;
  1713. if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
  1714. goto out;
  1715. r = 0;
  1716. break;
  1717. }
  1718. case KVM_SET_SREGS: {
  1719. kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
  1720. if (IS_ERR(kvm_sregs)) {
  1721. r = PTR_ERR(kvm_sregs);
  1722. kvm_sregs = NULL;
  1723. goto out;
  1724. }
  1725. r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
  1726. break;
  1727. }
  1728. case KVM_GET_MP_STATE: {
  1729. struct kvm_mp_state mp_state;
  1730. r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
  1731. if (r)
  1732. goto out;
  1733. r = -EFAULT;
  1734. if (copy_to_user(argp, &mp_state, sizeof mp_state))
  1735. goto out;
  1736. r = 0;
  1737. break;
  1738. }
  1739. case KVM_SET_MP_STATE: {
  1740. struct kvm_mp_state mp_state;
  1741. r = -EFAULT;
  1742. if (copy_from_user(&mp_state, argp, sizeof mp_state))
  1743. goto out;
  1744. r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
  1745. break;
  1746. }
  1747. case KVM_TRANSLATE: {
  1748. struct kvm_translation tr;
  1749. r = -EFAULT;
  1750. if (copy_from_user(&tr, argp, sizeof tr))
  1751. goto out;
  1752. r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
  1753. if (r)
  1754. goto out;
  1755. r = -EFAULT;
  1756. if (copy_to_user(argp, &tr, sizeof tr))
  1757. goto out;
  1758. r = 0;
  1759. break;
  1760. }
  1761. case KVM_SET_GUEST_DEBUG: {
  1762. struct kvm_guest_debug dbg;
  1763. r = -EFAULT;
  1764. if (copy_from_user(&dbg, argp, sizeof dbg))
  1765. goto out;
  1766. r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
  1767. break;
  1768. }
  1769. case KVM_SET_SIGNAL_MASK: {
  1770. struct kvm_signal_mask __user *sigmask_arg = argp;
  1771. struct kvm_signal_mask kvm_sigmask;
  1772. sigset_t sigset, *p;
  1773. p = NULL;
  1774. if (argp) {
  1775. r = -EFAULT;
  1776. if (copy_from_user(&kvm_sigmask, argp,
  1777. sizeof kvm_sigmask))
  1778. goto out;
  1779. r = -EINVAL;
  1780. if (kvm_sigmask.len != sizeof sigset)
  1781. goto out;
  1782. r = -EFAULT;
  1783. if (copy_from_user(&sigset, sigmask_arg->sigset,
  1784. sizeof sigset))
  1785. goto out;
  1786. p = &sigset;
  1787. }
  1788. r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
  1789. break;
  1790. }
  1791. case KVM_GET_FPU: {
  1792. fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
  1793. r = -ENOMEM;
  1794. if (!fpu)
  1795. goto out;
  1796. r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
  1797. if (r)
  1798. goto out;
  1799. r = -EFAULT;
  1800. if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
  1801. goto out;
  1802. r = 0;
  1803. break;
  1804. }
  1805. case KVM_SET_FPU: {
  1806. fpu = memdup_user(argp, sizeof(*fpu));
  1807. if (IS_ERR(fpu)) {
  1808. r = PTR_ERR(fpu);
  1809. fpu = NULL;
  1810. goto out;
  1811. }
  1812. r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
  1813. break;
  1814. }
  1815. default:
  1816. r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  1817. }
  1818. out:
  1819. vcpu_put(vcpu);
  1820. kfree(fpu);
  1821. kfree(kvm_sregs);
  1822. return r;
  1823. }
  1824. #ifdef CONFIG_COMPAT
  1825. static long kvm_vcpu_compat_ioctl(struct file *filp,
  1826. unsigned int ioctl, unsigned long arg)
  1827. {
  1828. struct kvm_vcpu *vcpu = filp->private_data;
  1829. void __user *argp = compat_ptr(arg);
  1830. int r;
  1831. if (vcpu->kvm->mm != current->mm)
  1832. return -EIO;
  1833. switch (ioctl) {
  1834. case KVM_SET_SIGNAL_MASK: {
  1835. struct kvm_signal_mask __user *sigmask_arg = argp;
  1836. struct kvm_signal_mask kvm_sigmask;
  1837. compat_sigset_t csigset;
  1838. sigset_t sigset;
  1839. if (argp) {
  1840. r = -EFAULT;
  1841. if (copy_from_user(&kvm_sigmask, argp,
  1842. sizeof kvm_sigmask))
  1843. goto out;
  1844. r = -EINVAL;
  1845. if (kvm_sigmask.len != sizeof csigset)
  1846. goto out;
  1847. r = -EFAULT;
  1848. if (copy_from_user(&csigset, sigmask_arg->sigset,
  1849. sizeof csigset))
  1850. goto out;
  1851. sigset_from_compat(&sigset, &csigset);
  1852. r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
  1853. } else
  1854. r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
  1855. break;
  1856. }
  1857. default:
  1858. r = kvm_vcpu_ioctl(filp, ioctl, arg);
  1859. }
  1860. out:
  1861. return r;
  1862. }
  1863. #endif
  1864. static int kvm_device_ioctl_attr(struct kvm_device *dev,
  1865. int (*accessor)(struct kvm_device *dev,
  1866. struct kvm_device_attr *attr),
  1867. unsigned long arg)
  1868. {
  1869. struct kvm_device_attr attr;
  1870. if (!accessor)
  1871. return -EPERM;
  1872. if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
  1873. return -EFAULT;
  1874. return accessor(dev, &attr);
  1875. }
  1876. static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
  1877. unsigned long arg)
  1878. {
  1879. struct kvm_device *dev = filp->private_data;
  1880. switch (ioctl) {
  1881. case KVM_SET_DEVICE_ATTR:
  1882. return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
  1883. case KVM_GET_DEVICE_ATTR:
  1884. return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
  1885. case KVM_HAS_DEVICE_ATTR:
  1886. return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
  1887. default:
  1888. if (dev->ops->ioctl)
  1889. return dev->ops->ioctl(dev, ioctl, arg);
  1890. return -ENOTTY;
  1891. }
  1892. }
  1893. static int kvm_device_release(struct inode *inode, struct file *filp)
  1894. {
  1895. struct kvm_device *dev = filp->private_data;
  1896. struct kvm *kvm = dev->kvm;
  1897. kvm_put_kvm(kvm);
  1898. return 0;
  1899. }
  1900. static const struct file_operations kvm_device_fops = {
  1901. .unlocked_ioctl = kvm_device_ioctl,
  1902. #ifdef CONFIG_COMPAT
  1903. .compat_ioctl = kvm_device_ioctl,
  1904. #endif
  1905. .release = kvm_device_release,
  1906. };
  1907. struct kvm_device *kvm_device_from_filp(struct file *filp)
  1908. {
  1909. if (filp->f_op != &kvm_device_fops)
  1910. return NULL;
  1911. return filp->private_data;
  1912. }
  1913. static int kvm_ioctl_create_device(struct kvm *kvm,
  1914. struct kvm_create_device *cd)
  1915. {
  1916. struct kvm_device_ops *ops = NULL;
  1917. struct kvm_device *dev;
  1918. bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
  1919. int ret;
  1920. switch (cd->type) {
  1921. #ifdef CONFIG_KVM_MPIC
  1922. case KVM_DEV_TYPE_FSL_MPIC_20:
  1923. case KVM_DEV_TYPE_FSL_MPIC_42:
  1924. ops = &kvm_mpic_ops;
  1925. break;
  1926. #endif
  1927. #ifdef CONFIG_KVM_XICS
  1928. case KVM_DEV_TYPE_XICS:
  1929. ops = &kvm_xics_ops;
  1930. break;
  1931. #endif
  1932. default:
  1933. return -ENODEV;
  1934. }
  1935. if (test)
  1936. return 0;
  1937. dev = kzalloc(sizeof(*dev), GFP_KERNEL);
  1938. if (!dev)
  1939. return -ENOMEM;
  1940. dev->ops = ops;
  1941. dev->kvm = kvm;
  1942. ret = ops->create(dev, cd->type);
  1943. if (ret < 0) {
  1944. kfree(dev);
  1945. return ret;
  1946. }
  1947. ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR);
  1948. if (ret < 0) {
  1949. ops->destroy(dev);
  1950. return ret;
  1951. }
  1952. list_add(&dev->vm_node, &kvm->devices);
  1953. kvm_get_kvm(kvm);
  1954. cd->fd = ret;
  1955. return 0;
  1956. }
  1957. static long kvm_vm_ioctl(struct file *filp,
  1958. unsigned int ioctl, unsigned long arg)
  1959. {
  1960. struct kvm *kvm = filp->private_data;
  1961. void __user *argp = (void __user *)arg;
  1962. int r;
  1963. if (kvm->mm != current->mm)
  1964. return -EIO;
  1965. switch (ioctl) {
  1966. case KVM_CREATE_VCPU:
  1967. r = kvm_vm_ioctl_create_vcpu(kvm, arg);
  1968. break;
  1969. case KVM_SET_USER_MEMORY_REGION: {
  1970. struct kvm_userspace_memory_region kvm_userspace_mem;
  1971. r = -EFAULT;
  1972. if (copy_from_user(&kvm_userspace_mem, argp,
  1973. sizeof kvm_userspace_mem))
  1974. goto out;
  1975. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
  1976. break;
  1977. }
  1978. case KVM_GET_DIRTY_LOG: {
  1979. struct kvm_dirty_log log;
  1980. r = -EFAULT;
  1981. if (copy_from_user(&log, argp, sizeof log))
  1982. goto out;
  1983. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  1984. break;
  1985. }
  1986. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  1987. case KVM_REGISTER_COALESCED_MMIO: {
  1988. struct kvm_coalesced_mmio_zone zone;
  1989. r = -EFAULT;
  1990. if (copy_from_user(&zone, argp, sizeof zone))
  1991. goto out;
  1992. r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
  1993. break;
  1994. }
  1995. case KVM_UNREGISTER_COALESCED_MMIO: {
  1996. struct kvm_coalesced_mmio_zone zone;
  1997. r = -EFAULT;
  1998. if (copy_from_user(&zone, argp, sizeof zone))
  1999. goto out;
  2000. r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
  2001. break;
  2002. }
  2003. #endif
  2004. case KVM_IRQFD: {
  2005. struct kvm_irqfd data;
  2006. r = -EFAULT;
  2007. if (copy_from_user(&data, argp, sizeof data))
  2008. goto out;
  2009. r = kvm_irqfd(kvm, &data);
  2010. break;
  2011. }
  2012. case KVM_IOEVENTFD: {
  2013. struct kvm_ioeventfd data;
  2014. r = -EFAULT;
  2015. if (copy_from_user(&data, argp, sizeof data))
  2016. goto out;
  2017. r = kvm_ioeventfd(kvm, &data);
  2018. break;
  2019. }
  2020. #ifdef CONFIG_KVM_APIC_ARCHITECTURE
  2021. case KVM_SET_BOOT_CPU_ID:
  2022. r = 0;
  2023. mutex_lock(&kvm->lock);
  2024. if (atomic_read(&kvm->online_vcpus) != 0)
  2025. r = -EBUSY;
  2026. else
  2027. kvm->bsp_vcpu_id = arg;
  2028. mutex_unlock(&kvm->lock);
  2029. break;
  2030. #endif
  2031. #ifdef CONFIG_HAVE_KVM_MSI
  2032. case KVM_SIGNAL_MSI: {
  2033. struct kvm_msi msi;
  2034. r = -EFAULT;
  2035. if (copy_from_user(&msi, argp, sizeof msi))
  2036. goto out;
  2037. r = kvm_send_userspace_msi(kvm, &msi);
  2038. break;
  2039. }
  2040. #endif
  2041. #ifdef __KVM_HAVE_IRQ_LINE
  2042. case KVM_IRQ_LINE_STATUS:
  2043. case KVM_IRQ_LINE: {
  2044. struct kvm_irq_level irq_event;
  2045. r = -EFAULT;
  2046. if (copy_from_user(&irq_event, argp, sizeof irq_event))
  2047. goto out;
  2048. r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
  2049. ioctl == KVM_IRQ_LINE_STATUS);
  2050. if (r)
  2051. goto out;
  2052. r = -EFAULT;
  2053. if (ioctl == KVM_IRQ_LINE_STATUS) {
  2054. if (copy_to_user(argp, &irq_event, sizeof irq_event))
  2055. goto out;
  2056. }
  2057. r = 0;
  2058. break;
  2059. }
  2060. #endif
  2061. #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
  2062. case KVM_SET_GSI_ROUTING: {
  2063. struct kvm_irq_routing routing;
  2064. struct kvm_irq_routing __user *urouting;
  2065. struct kvm_irq_routing_entry *entries;
  2066. r = -EFAULT;
  2067. if (copy_from_user(&routing, argp, sizeof(routing)))
  2068. goto out;
  2069. r = -EINVAL;
  2070. if (routing.nr >= KVM_MAX_IRQ_ROUTES)
  2071. goto out;
  2072. if (routing.flags)
  2073. goto out;
  2074. r = -ENOMEM;
  2075. entries = vmalloc(routing.nr * sizeof(*entries));
  2076. if (!entries)
  2077. goto out;
  2078. r = -EFAULT;
  2079. urouting = argp;
  2080. if (copy_from_user(entries, urouting->entries,
  2081. routing.nr * sizeof(*entries)))
  2082. goto out_free_irq_routing;
  2083. r = kvm_set_irq_routing(kvm, entries, routing.nr,
  2084. routing.flags);
  2085. out_free_irq_routing:
  2086. vfree(entries);
  2087. break;
  2088. }
  2089. #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
  2090. case KVM_CREATE_DEVICE: {
  2091. struct kvm_create_device cd;
  2092. r = -EFAULT;
  2093. if (copy_from_user(&cd, argp, sizeof(cd)))
  2094. goto out;
  2095. r = kvm_ioctl_create_device(kvm, &cd);
  2096. if (r)
  2097. goto out;
  2098. r = -EFAULT;
  2099. if (copy_to_user(argp, &cd, sizeof(cd)))
  2100. goto out;
  2101. r = 0;
  2102. break;
  2103. }
  2104. default:
  2105. r = kvm_arch_vm_ioctl(filp, ioctl, arg);
  2106. if (r == -ENOTTY)
  2107. r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
  2108. }
  2109. out:
  2110. return r;
  2111. }
  2112. #ifdef CONFIG_COMPAT
  2113. struct compat_kvm_dirty_log {
  2114. __u32 slot;
  2115. __u32 padding1;
  2116. union {
  2117. compat_uptr_t dirty_bitmap; /* one bit per page */
  2118. __u64 padding2;
  2119. };
  2120. };
  2121. static long kvm_vm_compat_ioctl(struct file *filp,
  2122. unsigned int ioctl, unsigned long arg)
  2123. {
  2124. struct kvm *kvm = filp->private_data;
  2125. int r;
  2126. if (kvm->mm != current->mm)
  2127. return -EIO;
  2128. switch (ioctl) {
  2129. case KVM_GET_DIRTY_LOG: {
  2130. struct compat_kvm_dirty_log compat_log;
  2131. struct kvm_dirty_log log;
  2132. r = -EFAULT;
  2133. if (copy_from_user(&compat_log, (void __user *)arg,
  2134. sizeof(compat_log)))
  2135. goto out;
  2136. log.slot = compat_log.slot;
  2137. log.padding1 = compat_log.padding1;
  2138. log.padding2 = compat_log.padding2;
  2139. log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
  2140. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  2141. break;
  2142. }
  2143. default:
  2144. r = kvm_vm_ioctl(filp, ioctl, arg);
  2145. }
  2146. out:
  2147. return r;
  2148. }
  2149. #endif
  2150. static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  2151. {
  2152. struct page *page[1];
  2153. unsigned long addr;
  2154. int npages;
  2155. gfn_t gfn = vmf->pgoff;
  2156. struct kvm *kvm = vma->vm_file->private_data;
  2157. addr = gfn_to_hva(kvm, gfn);
  2158. if (kvm_is_error_hva(addr))
  2159. return VM_FAULT_SIGBUS;
  2160. npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
  2161. NULL);
  2162. if (unlikely(npages != 1))
  2163. return VM_FAULT_SIGBUS;
  2164. vmf->page = page[0];
  2165. return 0;
  2166. }
  2167. static const struct vm_operations_struct kvm_vm_vm_ops = {
  2168. .fault = kvm_vm_fault,
  2169. };
  2170. static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
  2171. {
  2172. vma->vm_ops = &kvm_vm_vm_ops;
  2173. return 0;
  2174. }
  2175. static struct file_operations kvm_vm_fops = {
  2176. .release = kvm_vm_release,
  2177. .unlocked_ioctl = kvm_vm_ioctl,
  2178. #ifdef CONFIG_COMPAT
  2179. .compat_ioctl = kvm_vm_compat_ioctl,
  2180. #endif
  2181. .mmap = kvm_vm_mmap,
  2182. .llseek = noop_llseek,
  2183. };
  2184. static int kvm_dev_ioctl_create_vm(unsigned long type)
  2185. {
  2186. int r;
  2187. struct kvm *kvm;
  2188. kvm = kvm_create_vm(type);
  2189. if (IS_ERR(kvm))
  2190. return PTR_ERR(kvm);
  2191. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  2192. r = kvm_coalesced_mmio_init(kvm);
  2193. if (r < 0) {
  2194. kvm_put_kvm(kvm);
  2195. return r;
  2196. }
  2197. #endif
  2198. r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
  2199. if (r < 0)
  2200. kvm_put_kvm(kvm);
  2201. return r;
  2202. }
  2203. static long kvm_dev_ioctl_check_extension_generic(long arg)
  2204. {
  2205. switch (arg) {
  2206. case KVM_CAP_USER_MEMORY:
  2207. case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
  2208. case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
  2209. #ifdef CONFIG_KVM_APIC_ARCHITECTURE
  2210. case KVM_CAP_SET_BOOT_CPU_ID:
  2211. #endif
  2212. case KVM_CAP_INTERNAL_ERROR_DATA:
  2213. #ifdef CONFIG_HAVE_KVM_MSI
  2214. case KVM_CAP_SIGNAL_MSI:
  2215. #endif
  2216. #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
  2217. case KVM_CAP_IRQFD_RESAMPLE:
  2218. #endif
  2219. return 1;
  2220. #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
  2221. case KVM_CAP_IRQ_ROUTING:
  2222. return KVM_MAX_IRQ_ROUTES;
  2223. #endif
  2224. default:
  2225. break;
  2226. }
  2227. return kvm_dev_ioctl_check_extension(arg);
  2228. }
  2229. static long kvm_dev_ioctl(struct file *filp,
  2230. unsigned int ioctl, unsigned long arg)
  2231. {
  2232. long r = -EINVAL;
  2233. switch (ioctl) {
  2234. case KVM_GET_API_VERSION:
  2235. r = -EINVAL;
  2236. if (arg)
  2237. goto out;
  2238. r = KVM_API_VERSION;
  2239. break;
  2240. case KVM_CREATE_VM:
  2241. r = kvm_dev_ioctl_create_vm(arg);
  2242. break;
  2243. case KVM_CHECK_EXTENSION:
  2244. r = kvm_dev_ioctl_check_extension_generic(arg);
  2245. break;
  2246. case KVM_GET_VCPU_MMAP_SIZE:
  2247. r = -EINVAL;
  2248. if (arg)
  2249. goto out;
  2250. r = PAGE_SIZE; /* struct kvm_run */
  2251. #ifdef CONFIG_X86
  2252. r += PAGE_SIZE; /* pio data page */
  2253. #endif
  2254. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  2255. r += PAGE_SIZE; /* coalesced mmio ring page */
  2256. #endif
  2257. break;
  2258. case KVM_TRACE_ENABLE:
  2259. case KVM_TRACE_PAUSE:
  2260. case KVM_TRACE_DISABLE:
  2261. r = -EOPNOTSUPP;
  2262. break;
  2263. default:
  2264. return kvm_arch_dev_ioctl(filp, ioctl, arg);
  2265. }
  2266. out:
  2267. return r;
  2268. }
  2269. static struct file_operations kvm_chardev_ops = {
  2270. .unlocked_ioctl = kvm_dev_ioctl,
  2271. .compat_ioctl = kvm_dev_ioctl,
  2272. .llseek = noop_llseek,
  2273. };
  2274. static struct miscdevice kvm_dev = {
  2275. KVM_MINOR,
  2276. "kvm",
  2277. &kvm_chardev_ops,
  2278. };
  2279. static void hardware_enable_nolock(void *junk)
  2280. {
  2281. int cpu = raw_smp_processor_id();
  2282. int r;
  2283. if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
  2284. return;
  2285. cpumask_set_cpu(cpu, cpus_hardware_enabled);
  2286. r = kvm_arch_hardware_enable(NULL);
  2287. if (r) {
  2288. cpumask_clear_cpu(cpu, cpus_hardware_enabled);
  2289. atomic_inc(&hardware_enable_failed);
  2290. printk(KERN_INFO "kvm: enabling virtualization on "
  2291. "CPU%d failed\n", cpu);
  2292. }
  2293. }
  2294. static void hardware_enable(void *junk)
  2295. {
  2296. raw_spin_lock(&kvm_lock);
  2297. hardware_enable_nolock(junk);
  2298. raw_spin_unlock(&kvm_lock);
  2299. }
  2300. static void hardware_disable_nolock(void *junk)
  2301. {
  2302. int cpu = raw_smp_processor_id();
  2303. if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
  2304. return;
  2305. cpumask_clear_cpu(cpu, cpus_hardware_enabled);
  2306. kvm_arch_hardware_disable(NULL);
  2307. }
  2308. static void hardware_disable(void *junk)
  2309. {
  2310. raw_spin_lock(&kvm_lock);
  2311. hardware_disable_nolock(junk);
  2312. raw_spin_unlock(&kvm_lock);
  2313. }
  2314. static void hardware_disable_all_nolock(void)
  2315. {
  2316. BUG_ON(!kvm_usage_count);
  2317. kvm_usage_count--;
  2318. if (!kvm_usage_count)
  2319. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2320. }
  2321. static void hardware_disable_all(void)
  2322. {
  2323. raw_spin_lock(&kvm_lock);
  2324. hardware_disable_all_nolock();
  2325. raw_spin_unlock(&kvm_lock);
  2326. }
  2327. static int hardware_enable_all(void)
  2328. {
  2329. int r = 0;
  2330. raw_spin_lock(&kvm_lock);
  2331. kvm_usage_count++;
  2332. if (kvm_usage_count == 1) {
  2333. atomic_set(&hardware_enable_failed, 0);
  2334. on_each_cpu(hardware_enable_nolock, NULL, 1);
  2335. if (atomic_read(&hardware_enable_failed)) {
  2336. hardware_disable_all_nolock();
  2337. r = -EBUSY;
  2338. }
  2339. }
  2340. raw_spin_unlock(&kvm_lock);
  2341. return r;
  2342. }
  2343. static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  2344. void *v)
  2345. {
  2346. int cpu = (long)v;
  2347. if (!kvm_usage_count)
  2348. return NOTIFY_OK;
  2349. val &= ~CPU_TASKS_FROZEN;
  2350. switch (val) {
  2351. case CPU_DYING:
  2352. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  2353. cpu);
  2354. hardware_disable(NULL);
  2355. break;
  2356. case CPU_STARTING:
  2357. printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
  2358. cpu);
  2359. hardware_enable(NULL);
  2360. break;
  2361. }
  2362. return NOTIFY_OK;
  2363. }
  2364. static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
  2365. void *v)
  2366. {
  2367. /*
  2368. * Some (well, at least mine) BIOSes hang on reboot if
  2369. * in vmx root mode.
  2370. *
  2371. * And Intel TXT required VMX off for all cpu when system shutdown.
  2372. */
  2373. printk(KERN_INFO "kvm: exiting hardware virtualization\n");
  2374. kvm_rebooting = true;
  2375. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2376. return NOTIFY_OK;
  2377. }
  2378. static struct notifier_block kvm_reboot_notifier = {
  2379. .notifier_call = kvm_reboot,
  2380. .priority = 0,
  2381. };
  2382. static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
  2383. {
  2384. int i;
  2385. for (i = 0; i < bus->dev_count; i++) {
  2386. struct kvm_io_device *pos = bus->range[i].dev;
  2387. kvm_iodevice_destructor(pos);
  2388. }
  2389. kfree(bus);
  2390. }
  2391. static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
  2392. {
  2393. const struct kvm_io_range *r1 = p1;
  2394. const struct kvm_io_range *r2 = p2;
  2395. if (r1->addr < r2->addr)
  2396. return -1;
  2397. if (r1->addr + r1->len > r2->addr + r2->len)
  2398. return 1;
  2399. return 0;
  2400. }
  2401. static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
  2402. gpa_t addr, int len)
  2403. {
  2404. bus->range[bus->dev_count++] = (struct kvm_io_range) {
  2405. .addr = addr,
  2406. .len = len,
  2407. .dev = dev,
  2408. };
  2409. sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
  2410. kvm_io_bus_sort_cmp, NULL);
  2411. return 0;
  2412. }
  2413. static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
  2414. gpa_t addr, int len)
  2415. {
  2416. struct kvm_io_range *range, key;
  2417. int off;
  2418. key = (struct kvm_io_range) {
  2419. .addr = addr,
  2420. .len = len,
  2421. };
  2422. range = bsearch(&key, bus->range, bus->dev_count,
  2423. sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
  2424. if (range == NULL)
  2425. return -ENOENT;
  2426. off = range - bus->range;
  2427. while (off > 0 && kvm_io_bus_sort_cmp(&key, &bus->range[off-1]) == 0)
  2428. off--;
  2429. return off;
  2430. }
  2431. /* kvm_io_bus_write - called under kvm->slots_lock */
  2432. int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  2433. int len, const void *val)
  2434. {
  2435. int idx;
  2436. struct kvm_io_bus *bus;
  2437. struct kvm_io_range range;
  2438. range = (struct kvm_io_range) {
  2439. .addr = addr,
  2440. .len = len,
  2441. };
  2442. bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
  2443. idx = kvm_io_bus_get_first_dev(bus, addr, len);
  2444. if (idx < 0)
  2445. return -EOPNOTSUPP;
  2446. while (idx < bus->dev_count &&
  2447. kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
  2448. if (!kvm_iodevice_write(bus->range[idx].dev, addr, len, val))
  2449. return 0;
  2450. idx++;
  2451. }
  2452. return -EOPNOTSUPP;
  2453. }
  2454. /* kvm_io_bus_read - called under kvm->slots_lock */
  2455. int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  2456. int len, void *val)
  2457. {
  2458. int idx;
  2459. struct kvm_io_bus *bus;
  2460. struct kvm_io_range range;
  2461. range = (struct kvm_io_range) {
  2462. .addr = addr,
  2463. .len = len,
  2464. };
  2465. bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
  2466. idx = kvm_io_bus_get_first_dev(bus, addr, len);
  2467. if (idx < 0)
  2468. return -EOPNOTSUPP;
  2469. while (idx < bus->dev_count &&
  2470. kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
  2471. if (!kvm_iodevice_read(bus->range[idx].dev, addr, len, val))
  2472. return 0;
  2473. idx++;
  2474. }
  2475. return -EOPNOTSUPP;
  2476. }
  2477. /* Caller must hold slots_lock. */
  2478. int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  2479. int len, struct kvm_io_device *dev)
  2480. {
  2481. struct kvm_io_bus *new_bus, *bus;
  2482. bus = kvm->buses[bus_idx];
  2483. if (bus->dev_count > NR_IOBUS_DEVS - 1)
  2484. return -ENOSPC;
  2485. new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
  2486. sizeof(struct kvm_io_range)), GFP_KERNEL);
  2487. if (!new_bus)
  2488. return -ENOMEM;
  2489. memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
  2490. sizeof(struct kvm_io_range)));
  2491. kvm_io_bus_insert_dev(new_bus, dev, addr, len);
  2492. rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
  2493. synchronize_srcu_expedited(&kvm->srcu);
  2494. kfree(bus);
  2495. return 0;
  2496. }
  2497. /* Caller must hold slots_lock. */
  2498. int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
  2499. struct kvm_io_device *dev)
  2500. {
  2501. int i, r;
  2502. struct kvm_io_bus *new_bus, *bus;
  2503. bus = kvm->buses[bus_idx];
  2504. r = -ENOENT;
  2505. for (i = 0; i < bus->dev_count; i++)
  2506. if (bus->range[i].dev == dev) {
  2507. r = 0;
  2508. break;
  2509. }
  2510. if (r)
  2511. return r;
  2512. new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
  2513. sizeof(struct kvm_io_range)), GFP_KERNEL);
  2514. if (!new_bus)
  2515. return -ENOMEM;
  2516. memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
  2517. new_bus->dev_count--;
  2518. memcpy(new_bus->range + i, bus->range + i + 1,
  2519. (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
  2520. rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
  2521. synchronize_srcu_expedited(&kvm->srcu);
  2522. kfree(bus);
  2523. return r;
  2524. }
  2525. static struct notifier_block kvm_cpu_notifier = {
  2526. .notifier_call = kvm_cpu_hotplug,
  2527. };
  2528. static int vm_stat_get(void *_offset, u64 *val)
  2529. {
  2530. unsigned offset = (long)_offset;
  2531. struct kvm *kvm;
  2532. *val = 0;
  2533. raw_spin_lock(&kvm_lock);
  2534. list_for_each_entry(kvm, &vm_list, vm_list)
  2535. *val += *(u32 *)((void *)kvm + offset);
  2536. raw_spin_unlock(&kvm_lock);
  2537. return 0;
  2538. }
  2539. DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
  2540. static int vcpu_stat_get(void *_offset, u64 *val)
  2541. {
  2542. unsigned offset = (long)_offset;
  2543. struct kvm *kvm;
  2544. struct kvm_vcpu *vcpu;
  2545. int i;
  2546. *val = 0;
  2547. raw_spin_lock(&kvm_lock);
  2548. list_for_each_entry(kvm, &vm_list, vm_list)
  2549. kvm_for_each_vcpu(i, vcpu, kvm)
  2550. *val += *(u32 *)((void *)vcpu + offset);
  2551. raw_spin_unlock(&kvm_lock);
  2552. return 0;
  2553. }
  2554. DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
  2555. static const struct file_operations *stat_fops[] = {
  2556. [KVM_STAT_VCPU] = &vcpu_stat_fops,
  2557. [KVM_STAT_VM] = &vm_stat_fops,
  2558. };
  2559. static int kvm_init_debug(void)
  2560. {
  2561. int r = -EFAULT;
  2562. struct kvm_stats_debugfs_item *p;
  2563. kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
  2564. if (kvm_debugfs_dir == NULL)
  2565. goto out;
  2566. for (p = debugfs_entries; p->name; ++p) {
  2567. p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
  2568. (void *)(long)p->offset,
  2569. stat_fops[p->kind]);
  2570. if (p->dentry == NULL)
  2571. goto out_dir;
  2572. }
  2573. return 0;
  2574. out_dir:
  2575. debugfs_remove_recursive(kvm_debugfs_dir);
  2576. out:
  2577. return r;
  2578. }
  2579. static void kvm_exit_debug(void)
  2580. {
  2581. struct kvm_stats_debugfs_item *p;
  2582. for (p = debugfs_entries; p->name; ++p)
  2583. debugfs_remove(p->dentry);
  2584. debugfs_remove(kvm_debugfs_dir);
  2585. }
  2586. static int kvm_suspend(void)
  2587. {
  2588. if (kvm_usage_count)
  2589. hardware_disable_nolock(NULL);
  2590. return 0;
  2591. }
  2592. static void kvm_resume(void)
  2593. {
  2594. if (kvm_usage_count) {
  2595. WARN_ON(raw_spin_is_locked(&kvm_lock));
  2596. hardware_enable_nolock(NULL);
  2597. }
  2598. }
  2599. static struct syscore_ops kvm_syscore_ops = {
  2600. .suspend = kvm_suspend,
  2601. .resume = kvm_resume,
  2602. };
  2603. static inline
  2604. struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
  2605. {
  2606. return container_of(pn, struct kvm_vcpu, preempt_notifier);
  2607. }
  2608. static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
  2609. {
  2610. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  2611. if (vcpu->preempted)
  2612. vcpu->preempted = false;
  2613. kvm_arch_vcpu_load(vcpu, cpu);
  2614. }
  2615. static void kvm_sched_out(struct preempt_notifier *pn,
  2616. struct task_struct *next)
  2617. {
  2618. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  2619. if (current->state == TASK_RUNNING)
  2620. vcpu->preempted = true;
  2621. kvm_arch_vcpu_put(vcpu);
  2622. }
  2623. int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
  2624. struct module *module)
  2625. {
  2626. int r;
  2627. int cpu;
  2628. r = kvm_arch_init(opaque);
  2629. if (r)
  2630. goto out_fail;
  2631. /*
  2632. * kvm_arch_init makes sure there's at most one caller
  2633. * for architectures that support multiple implementations,
  2634. * like intel and amd on x86.
  2635. * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
  2636. * conflicts in case kvm is already setup for another implementation.
  2637. */
  2638. r = kvm_irqfd_init();
  2639. if (r)
  2640. goto out_irqfd;
  2641. if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
  2642. r = -ENOMEM;
  2643. goto out_free_0;
  2644. }
  2645. r = kvm_arch_hardware_setup();
  2646. if (r < 0)
  2647. goto out_free_0a;
  2648. for_each_online_cpu(cpu) {
  2649. smp_call_function_single(cpu,
  2650. kvm_arch_check_processor_compat,
  2651. &r, 1);
  2652. if (r < 0)
  2653. goto out_free_1;
  2654. }
  2655. r = register_cpu_notifier(&kvm_cpu_notifier);
  2656. if (r)
  2657. goto out_free_2;
  2658. register_reboot_notifier(&kvm_reboot_notifier);
  2659. /* A kmem cache lets us meet the alignment requirements of fx_save. */
  2660. if (!vcpu_align)
  2661. vcpu_align = __alignof__(struct kvm_vcpu);
  2662. kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
  2663. 0, NULL);
  2664. if (!kvm_vcpu_cache) {
  2665. r = -ENOMEM;
  2666. goto out_free_3;
  2667. }
  2668. r = kvm_async_pf_init();
  2669. if (r)
  2670. goto out_free;
  2671. kvm_chardev_ops.owner = module;
  2672. kvm_vm_fops.owner = module;
  2673. kvm_vcpu_fops.owner = module;
  2674. r = misc_register(&kvm_dev);
  2675. if (r) {
  2676. printk(KERN_ERR "kvm: misc device register failed\n");
  2677. goto out_unreg;
  2678. }
  2679. register_syscore_ops(&kvm_syscore_ops);
  2680. kvm_preempt_ops.sched_in = kvm_sched_in;
  2681. kvm_preempt_ops.sched_out = kvm_sched_out;
  2682. r = kvm_init_debug();
  2683. if (r) {
  2684. printk(KERN_ERR "kvm: create debugfs files failed\n");
  2685. goto out_undebugfs;
  2686. }
  2687. return 0;
  2688. out_undebugfs:
  2689. unregister_syscore_ops(&kvm_syscore_ops);
  2690. out_unreg:
  2691. kvm_async_pf_deinit();
  2692. out_free:
  2693. kmem_cache_destroy(kvm_vcpu_cache);
  2694. out_free_3:
  2695. unregister_reboot_notifier(&kvm_reboot_notifier);
  2696. unregister_cpu_notifier(&kvm_cpu_notifier);
  2697. out_free_2:
  2698. out_free_1:
  2699. kvm_arch_hardware_unsetup();
  2700. out_free_0a:
  2701. free_cpumask_var(cpus_hardware_enabled);
  2702. out_free_0:
  2703. kvm_irqfd_exit();
  2704. out_irqfd:
  2705. kvm_arch_exit();
  2706. out_fail:
  2707. return r;
  2708. }
  2709. EXPORT_SYMBOL_GPL(kvm_init);
  2710. void kvm_exit(void)
  2711. {
  2712. kvm_exit_debug();
  2713. misc_deregister(&kvm_dev);
  2714. kmem_cache_destroy(kvm_vcpu_cache);
  2715. kvm_async_pf_deinit();
  2716. unregister_syscore_ops(&kvm_syscore_ops);
  2717. unregister_reboot_notifier(&kvm_reboot_notifier);
  2718. unregister_cpu_notifier(&kvm_cpu_notifier);
  2719. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2720. kvm_arch_hardware_unsetup();
  2721. kvm_arch_exit();
  2722. kvm_irqfd_exit();
  2723. free_cpumask_var(cpus_hardware_enabled);
  2724. }
  2725. EXPORT_SYMBOL_GPL(kvm_exit);