kvm_main.c 71 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  9. *
  10. * Authors:
  11. * Avi Kivity <avi@qumranet.com>
  12. * Yaniv Kamay <yaniv@qumranet.com>
  13. *
  14. * This work is licensed under the terms of the GNU GPL, version 2. See
  15. * the COPYING file in the top-level directory.
  16. *
  17. */
  18. #include "iodev.h"
  19. #include <linux/kvm_host.h>
  20. #include <linux/kvm.h>
  21. #include <linux/module.h>
  22. #include <linux/errno.h>
  23. #include <linux/percpu.h>
  24. #include <linux/mm.h>
  25. #include <linux/miscdevice.h>
  26. #include <linux/vmalloc.h>
  27. #include <linux/reboot.h>
  28. #include <linux/debugfs.h>
  29. #include <linux/highmem.h>
  30. #include <linux/file.h>
  31. #include <linux/syscore_ops.h>
  32. #include <linux/cpu.h>
  33. #include <linux/sched.h>
  34. #include <linux/cpumask.h>
  35. #include <linux/smp.h>
  36. #include <linux/anon_inodes.h>
  37. #include <linux/profile.h>
  38. #include <linux/kvm_para.h>
  39. #include <linux/pagemap.h>
  40. #include <linux/mman.h>
  41. #include <linux/swap.h>
  42. #include <linux/bitops.h>
  43. #include <linux/spinlock.h>
  44. #include <linux/compat.h>
  45. #include <linux/srcu.h>
  46. #include <linux/hugetlb.h>
  47. #include <linux/slab.h>
  48. #include <linux/sort.h>
  49. #include <linux/bsearch.h>
  50. #include <asm/processor.h>
  51. #include <asm/io.h>
  52. #include <asm/uaccess.h>
  53. #include <asm/pgtable.h>
  54. #include "coalesced_mmio.h"
  55. #include "async_pf.h"
  56. #define CREATE_TRACE_POINTS
  57. #include <trace/events/kvm.h>
  58. MODULE_AUTHOR("Qumranet");
  59. MODULE_LICENSE("GPL");
  60. /*
  61. * Ordering of locks:
  62. *
  63. * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
  64. */
  65. DEFINE_RAW_SPINLOCK(kvm_lock);
  66. LIST_HEAD(vm_list);
  67. static cpumask_var_t cpus_hardware_enabled;
  68. static int kvm_usage_count = 0;
  69. static atomic_t hardware_enable_failed;
  70. struct kmem_cache *kvm_vcpu_cache;
  71. EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  72. static __read_mostly struct preempt_ops kvm_preempt_ops;
  73. struct dentry *kvm_debugfs_dir;
  74. static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  75. unsigned long arg);
  76. #ifdef CONFIG_COMPAT
  77. static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
  78. unsigned long arg);
  79. #endif
  80. static int hardware_enable_all(void);
  81. static void hardware_disable_all(void);
  82. static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
  83. bool kvm_rebooting;
  84. EXPORT_SYMBOL_GPL(kvm_rebooting);
  85. static bool largepages_enabled = true;
  86. bool kvm_is_mmio_pfn(pfn_t pfn)
  87. {
  88. if (pfn_valid(pfn)) {
  89. int reserved;
  90. struct page *tail = pfn_to_page(pfn);
  91. struct page *head = compound_trans_head(tail);
  92. reserved = PageReserved(head);
  93. if (head != tail) {
  94. /*
  95. * "head" is not a dangling pointer
  96. * (compound_trans_head takes care of that)
  97. * but the hugepage may have been splitted
  98. * from under us (and we may not hold a
  99. * reference count on the head page so it can
  100. * be reused before we run PageReferenced), so
  101. * we've to check PageTail before returning
  102. * what we just read.
  103. */
  104. smp_rmb();
  105. if (PageTail(tail))
  106. return reserved;
  107. }
  108. return PageReserved(tail);
  109. }
  110. return true;
  111. }
  112. /*
  113. * Switches to specified vcpu, until a matching vcpu_put()
  114. */
  115. int vcpu_load(struct kvm_vcpu *vcpu)
  116. {
  117. int cpu;
  118. if (mutex_lock_killable(&vcpu->mutex))
  119. return -EINTR;
  120. if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
  121. /* The thread running this VCPU changed. */
  122. struct pid *oldpid = vcpu->pid;
  123. struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
  124. rcu_assign_pointer(vcpu->pid, newpid);
  125. synchronize_rcu();
  126. put_pid(oldpid);
  127. }
  128. cpu = get_cpu();
  129. preempt_notifier_register(&vcpu->preempt_notifier);
  130. kvm_arch_vcpu_load(vcpu, cpu);
  131. put_cpu();
  132. return 0;
  133. }
  134. void vcpu_put(struct kvm_vcpu *vcpu)
  135. {
  136. preempt_disable();
  137. kvm_arch_vcpu_put(vcpu);
  138. preempt_notifier_unregister(&vcpu->preempt_notifier);
  139. preempt_enable();
  140. mutex_unlock(&vcpu->mutex);
  141. }
  142. static void ack_flush(void *_completed)
  143. {
  144. }
  145. static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
  146. {
  147. int i, cpu, me;
  148. cpumask_var_t cpus;
  149. bool called = true;
  150. struct kvm_vcpu *vcpu;
  151. zalloc_cpumask_var(&cpus, GFP_ATOMIC);
  152. me = get_cpu();
  153. kvm_for_each_vcpu(i, vcpu, kvm) {
  154. kvm_make_request(req, vcpu);
  155. cpu = vcpu->cpu;
  156. /* Set ->requests bit before we read ->mode */
  157. smp_mb();
  158. if (cpus != NULL && cpu != -1 && cpu != me &&
  159. kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
  160. cpumask_set_cpu(cpu, cpus);
  161. }
  162. if (unlikely(cpus == NULL))
  163. smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
  164. else if (!cpumask_empty(cpus))
  165. smp_call_function_many(cpus, ack_flush, NULL, 1);
  166. else
  167. called = false;
  168. put_cpu();
  169. free_cpumask_var(cpus);
  170. return called;
  171. }
  172. void kvm_flush_remote_tlbs(struct kvm *kvm)
  173. {
  174. long dirty_count = kvm->tlbs_dirty;
  175. smp_mb();
  176. if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
  177. ++kvm->stat.remote_tlb_flush;
  178. cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
  179. }
  180. void kvm_reload_remote_mmus(struct kvm *kvm)
  181. {
  182. make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
  183. }
  184. void kvm_make_mclock_inprogress_request(struct kvm *kvm)
  185. {
  186. make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
  187. }
  188. void kvm_make_scan_ioapic_request(struct kvm *kvm)
  189. {
  190. make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
  191. }
  192. int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
  193. {
  194. struct page *page;
  195. int r;
  196. mutex_init(&vcpu->mutex);
  197. vcpu->cpu = -1;
  198. vcpu->kvm = kvm;
  199. vcpu->vcpu_id = id;
  200. vcpu->pid = NULL;
  201. init_waitqueue_head(&vcpu->wq);
  202. kvm_async_pf_vcpu_init(vcpu);
  203. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  204. if (!page) {
  205. r = -ENOMEM;
  206. goto fail;
  207. }
  208. vcpu->run = page_address(page);
  209. kvm_vcpu_set_in_spin_loop(vcpu, false);
  210. kvm_vcpu_set_dy_eligible(vcpu, false);
  211. vcpu->preempted = false;
  212. r = kvm_arch_vcpu_init(vcpu);
  213. if (r < 0)
  214. goto fail_free_run;
  215. return 0;
  216. fail_free_run:
  217. free_page((unsigned long)vcpu->run);
  218. fail:
  219. return r;
  220. }
  221. EXPORT_SYMBOL_GPL(kvm_vcpu_init);
  222. void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
  223. {
  224. put_pid(vcpu->pid);
  225. kvm_arch_vcpu_uninit(vcpu);
  226. free_page((unsigned long)vcpu->run);
  227. }
  228. EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
  229. #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
  230. static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
  231. {
  232. return container_of(mn, struct kvm, mmu_notifier);
  233. }
  234. static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
  235. struct mm_struct *mm,
  236. unsigned long address)
  237. {
  238. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  239. int need_tlb_flush, idx;
  240. /*
  241. * When ->invalidate_page runs, the linux pte has been zapped
  242. * already but the page is still allocated until
  243. * ->invalidate_page returns. So if we increase the sequence
  244. * here the kvm page fault will notice if the spte can't be
  245. * established because the page is going to be freed. If
  246. * instead the kvm page fault establishes the spte before
  247. * ->invalidate_page runs, kvm_unmap_hva will release it
  248. * before returning.
  249. *
  250. * The sequence increase only need to be seen at spin_unlock
  251. * time, and not at spin_lock time.
  252. *
  253. * Increasing the sequence after the spin_unlock would be
  254. * unsafe because the kvm page fault could then establish the
  255. * pte after kvm_unmap_hva returned, without noticing the page
  256. * is going to be freed.
  257. */
  258. idx = srcu_read_lock(&kvm->srcu);
  259. spin_lock(&kvm->mmu_lock);
  260. kvm->mmu_notifier_seq++;
  261. need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
  262. /* we've to flush the tlb before the pages can be freed */
  263. if (need_tlb_flush)
  264. kvm_flush_remote_tlbs(kvm);
  265. spin_unlock(&kvm->mmu_lock);
  266. srcu_read_unlock(&kvm->srcu, idx);
  267. }
  268. static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
  269. struct mm_struct *mm,
  270. unsigned long address,
  271. pte_t pte)
  272. {
  273. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  274. int idx;
  275. idx = srcu_read_lock(&kvm->srcu);
  276. spin_lock(&kvm->mmu_lock);
  277. kvm->mmu_notifier_seq++;
  278. kvm_set_spte_hva(kvm, address, pte);
  279. spin_unlock(&kvm->mmu_lock);
  280. srcu_read_unlock(&kvm->srcu, idx);
  281. }
  282. static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
  283. struct mm_struct *mm,
  284. unsigned long start,
  285. unsigned long end)
  286. {
  287. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  288. int need_tlb_flush = 0, idx;
  289. idx = srcu_read_lock(&kvm->srcu);
  290. spin_lock(&kvm->mmu_lock);
  291. /*
  292. * The count increase must become visible at unlock time as no
  293. * spte can be established without taking the mmu_lock and
  294. * count is also read inside the mmu_lock critical section.
  295. */
  296. kvm->mmu_notifier_count++;
  297. need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
  298. need_tlb_flush |= kvm->tlbs_dirty;
  299. /* we've to flush the tlb before the pages can be freed */
  300. if (need_tlb_flush)
  301. kvm_flush_remote_tlbs(kvm);
  302. spin_unlock(&kvm->mmu_lock);
  303. srcu_read_unlock(&kvm->srcu, idx);
  304. }
  305. static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
  306. struct mm_struct *mm,
  307. unsigned long start,
  308. unsigned long end)
  309. {
  310. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  311. spin_lock(&kvm->mmu_lock);
  312. /*
  313. * This sequence increase will notify the kvm page fault that
  314. * the page that is going to be mapped in the spte could have
  315. * been freed.
  316. */
  317. kvm->mmu_notifier_seq++;
  318. smp_wmb();
  319. /*
  320. * The above sequence increase must be visible before the
  321. * below count decrease, which is ensured by the smp_wmb above
  322. * in conjunction with the smp_rmb in mmu_notifier_retry().
  323. */
  324. kvm->mmu_notifier_count--;
  325. spin_unlock(&kvm->mmu_lock);
  326. BUG_ON(kvm->mmu_notifier_count < 0);
  327. }
  328. static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
  329. struct mm_struct *mm,
  330. unsigned long address)
  331. {
  332. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  333. int young, idx;
  334. idx = srcu_read_lock(&kvm->srcu);
  335. spin_lock(&kvm->mmu_lock);
  336. young = kvm_age_hva(kvm, address);
  337. if (young)
  338. kvm_flush_remote_tlbs(kvm);
  339. spin_unlock(&kvm->mmu_lock);
  340. srcu_read_unlock(&kvm->srcu, idx);
  341. return young;
  342. }
  343. static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
  344. struct mm_struct *mm,
  345. unsigned long address)
  346. {
  347. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  348. int young, idx;
  349. idx = srcu_read_lock(&kvm->srcu);
  350. spin_lock(&kvm->mmu_lock);
  351. young = kvm_test_age_hva(kvm, address);
  352. spin_unlock(&kvm->mmu_lock);
  353. srcu_read_unlock(&kvm->srcu, idx);
  354. return young;
  355. }
  356. static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
  357. struct mm_struct *mm)
  358. {
  359. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  360. int idx;
  361. idx = srcu_read_lock(&kvm->srcu);
  362. kvm_arch_flush_shadow_all(kvm);
  363. srcu_read_unlock(&kvm->srcu, idx);
  364. }
  365. static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
  366. .invalidate_page = kvm_mmu_notifier_invalidate_page,
  367. .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
  368. .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
  369. .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
  370. .test_young = kvm_mmu_notifier_test_young,
  371. .change_pte = kvm_mmu_notifier_change_pte,
  372. .release = kvm_mmu_notifier_release,
  373. };
  374. static int kvm_init_mmu_notifier(struct kvm *kvm)
  375. {
  376. kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
  377. return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
  378. }
  379. #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
  380. static int kvm_init_mmu_notifier(struct kvm *kvm)
  381. {
  382. return 0;
  383. }
  384. #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
  385. static void kvm_init_memslots_id(struct kvm *kvm)
  386. {
  387. int i;
  388. struct kvm_memslots *slots = kvm->memslots;
  389. for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
  390. slots->id_to_index[i] = slots->memslots[i].id = i;
  391. }
  392. static struct kvm *kvm_create_vm(unsigned long type)
  393. {
  394. int r, i;
  395. struct kvm *kvm = kvm_arch_alloc_vm();
  396. if (!kvm)
  397. return ERR_PTR(-ENOMEM);
  398. r = kvm_arch_init_vm(kvm, type);
  399. if (r)
  400. goto out_err_nodisable;
  401. r = hardware_enable_all();
  402. if (r)
  403. goto out_err_nodisable;
  404. #ifdef CONFIG_HAVE_KVM_IRQCHIP
  405. INIT_HLIST_HEAD(&kvm->mask_notifier_list);
  406. INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
  407. #endif
  408. BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
  409. r = -ENOMEM;
  410. kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
  411. if (!kvm->memslots)
  412. goto out_err_nosrcu;
  413. kvm_init_memslots_id(kvm);
  414. if (init_srcu_struct(&kvm->srcu))
  415. goto out_err_nosrcu;
  416. for (i = 0; i < KVM_NR_BUSES; i++) {
  417. kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
  418. GFP_KERNEL);
  419. if (!kvm->buses[i])
  420. goto out_err;
  421. }
  422. spin_lock_init(&kvm->mmu_lock);
  423. kvm->mm = current->mm;
  424. atomic_inc(&kvm->mm->mm_count);
  425. kvm_eventfd_init(kvm);
  426. mutex_init(&kvm->lock);
  427. mutex_init(&kvm->irq_lock);
  428. mutex_init(&kvm->slots_lock);
  429. atomic_set(&kvm->users_count, 1);
  430. INIT_LIST_HEAD(&kvm->devices);
  431. r = kvm_init_mmu_notifier(kvm);
  432. if (r)
  433. goto out_err;
  434. raw_spin_lock(&kvm_lock);
  435. list_add(&kvm->vm_list, &vm_list);
  436. raw_spin_unlock(&kvm_lock);
  437. return kvm;
  438. out_err:
  439. cleanup_srcu_struct(&kvm->srcu);
  440. out_err_nosrcu:
  441. hardware_disable_all();
  442. out_err_nodisable:
  443. for (i = 0; i < KVM_NR_BUSES; i++)
  444. kfree(kvm->buses[i]);
  445. kfree(kvm->memslots);
  446. kvm_arch_free_vm(kvm);
  447. return ERR_PTR(r);
  448. }
  449. /*
  450. * Avoid using vmalloc for a small buffer.
  451. * Should not be used when the size is statically known.
  452. */
  453. void *kvm_kvzalloc(unsigned long size)
  454. {
  455. if (size > PAGE_SIZE)
  456. return vzalloc(size);
  457. else
  458. return kzalloc(size, GFP_KERNEL);
  459. }
  460. void kvm_kvfree(const void *addr)
  461. {
  462. if (is_vmalloc_addr(addr))
  463. vfree(addr);
  464. else
  465. kfree(addr);
  466. }
  467. static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
  468. {
  469. if (!memslot->dirty_bitmap)
  470. return;
  471. kvm_kvfree(memslot->dirty_bitmap);
  472. memslot->dirty_bitmap = NULL;
  473. }
  474. /*
  475. * Free any memory in @free but not in @dont.
  476. */
  477. static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
  478. struct kvm_memory_slot *dont)
  479. {
  480. if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
  481. kvm_destroy_dirty_bitmap(free);
  482. kvm_arch_free_memslot(free, dont);
  483. free->npages = 0;
  484. }
  485. void kvm_free_physmem(struct kvm *kvm)
  486. {
  487. struct kvm_memslots *slots = kvm->memslots;
  488. struct kvm_memory_slot *memslot;
  489. kvm_for_each_memslot(memslot, slots)
  490. kvm_free_physmem_slot(memslot, NULL);
  491. kfree(kvm->memslots);
  492. }
  493. static void kvm_destroy_devices(struct kvm *kvm)
  494. {
  495. struct list_head *node, *tmp;
  496. list_for_each_safe(node, tmp, &kvm->devices) {
  497. struct kvm_device *dev =
  498. list_entry(node, struct kvm_device, vm_node);
  499. list_del(node);
  500. dev->ops->destroy(dev);
  501. }
  502. }
  503. static void kvm_destroy_vm(struct kvm *kvm)
  504. {
  505. int i;
  506. struct mm_struct *mm = kvm->mm;
  507. kvm_arch_sync_events(kvm);
  508. raw_spin_lock(&kvm_lock);
  509. list_del(&kvm->vm_list);
  510. raw_spin_unlock(&kvm_lock);
  511. kvm_free_irq_routing(kvm);
  512. for (i = 0; i < KVM_NR_BUSES; i++)
  513. kvm_io_bus_destroy(kvm->buses[i]);
  514. kvm_coalesced_mmio_free(kvm);
  515. #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
  516. mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
  517. #else
  518. kvm_arch_flush_shadow_all(kvm);
  519. #endif
  520. kvm_arch_destroy_vm(kvm);
  521. kvm_destroy_devices(kvm);
  522. kvm_free_physmem(kvm);
  523. cleanup_srcu_struct(&kvm->srcu);
  524. kvm_arch_free_vm(kvm);
  525. hardware_disable_all();
  526. mmdrop(mm);
  527. }
  528. void kvm_get_kvm(struct kvm *kvm)
  529. {
  530. atomic_inc(&kvm->users_count);
  531. }
  532. EXPORT_SYMBOL_GPL(kvm_get_kvm);
  533. void kvm_put_kvm(struct kvm *kvm)
  534. {
  535. if (atomic_dec_and_test(&kvm->users_count))
  536. kvm_destroy_vm(kvm);
  537. }
  538. EXPORT_SYMBOL_GPL(kvm_put_kvm);
  539. static int kvm_vm_release(struct inode *inode, struct file *filp)
  540. {
  541. struct kvm *kvm = filp->private_data;
  542. kvm_irqfd_release(kvm);
  543. kvm_put_kvm(kvm);
  544. return 0;
  545. }
  546. /*
  547. * Allocation size is twice as large as the actual dirty bitmap size.
  548. * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
  549. */
  550. static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
  551. {
  552. #ifndef CONFIG_S390
  553. unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
  554. memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
  555. if (!memslot->dirty_bitmap)
  556. return -ENOMEM;
  557. #endif /* !CONFIG_S390 */
  558. return 0;
  559. }
  560. static int cmp_memslot(const void *slot1, const void *slot2)
  561. {
  562. struct kvm_memory_slot *s1, *s2;
  563. s1 = (struct kvm_memory_slot *)slot1;
  564. s2 = (struct kvm_memory_slot *)slot2;
  565. if (s1->npages < s2->npages)
  566. return 1;
  567. if (s1->npages > s2->npages)
  568. return -1;
  569. return 0;
  570. }
  571. /*
  572. * Sort the memslots base on its size, so the larger slots
  573. * will get better fit.
  574. */
  575. static void sort_memslots(struct kvm_memslots *slots)
  576. {
  577. int i;
  578. sort(slots->memslots, KVM_MEM_SLOTS_NUM,
  579. sizeof(struct kvm_memory_slot), cmp_memslot, NULL);
  580. for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
  581. slots->id_to_index[slots->memslots[i].id] = i;
  582. }
  583. void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new,
  584. u64 last_generation)
  585. {
  586. if (new) {
  587. int id = new->id;
  588. struct kvm_memory_slot *old = id_to_memslot(slots, id);
  589. unsigned long npages = old->npages;
  590. *old = *new;
  591. if (new->npages != npages)
  592. sort_memslots(slots);
  593. }
  594. slots->generation = last_generation + 1;
  595. }
  596. static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
  597. {
  598. u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
  599. #ifdef KVM_CAP_READONLY_MEM
  600. valid_flags |= KVM_MEM_READONLY;
  601. #endif
  602. if (mem->flags & ~valid_flags)
  603. return -EINVAL;
  604. return 0;
  605. }
  606. static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
  607. struct kvm_memslots *slots, struct kvm_memory_slot *new)
  608. {
  609. struct kvm_memslots *old_memslots = kvm->memslots;
  610. update_memslots(slots, new, kvm->memslots->generation);
  611. rcu_assign_pointer(kvm->memslots, slots);
  612. synchronize_srcu_expedited(&kvm->srcu);
  613. return old_memslots;
  614. }
  615. /*
  616. * Allocate some memory and give it an address in the guest physical address
  617. * space.
  618. *
  619. * Discontiguous memory is allowed, mostly for framebuffers.
  620. *
  621. * Must be called holding mmap_sem for write.
  622. */
  623. int __kvm_set_memory_region(struct kvm *kvm,
  624. struct kvm_userspace_memory_region *mem)
  625. {
  626. int r;
  627. gfn_t base_gfn;
  628. unsigned long npages;
  629. struct kvm_memory_slot *slot;
  630. struct kvm_memory_slot old, new;
  631. struct kvm_memslots *slots = NULL, *old_memslots;
  632. enum kvm_mr_change change;
  633. r = check_memory_region_flags(mem);
  634. if (r)
  635. goto out;
  636. r = -EINVAL;
  637. /* General sanity checks */
  638. if (mem->memory_size & (PAGE_SIZE - 1))
  639. goto out;
  640. if (mem->guest_phys_addr & (PAGE_SIZE - 1))
  641. goto out;
  642. /* We can read the guest memory with __xxx_user() later on. */
  643. if ((mem->slot < KVM_USER_MEM_SLOTS) &&
  644. ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
  645. !access_ok(VERIFY_WRITE,
  646. (void __user *)(unsigned long)mem->userspace_addr,
  647. mem->memory_size)))
  648. goto out;
  649. if (mem->slot >= KVM_MEM_SLOTS_NUM)
  650. goto out;
  651. if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
  652. goto out;
  653. slot = id_to_memslot(kvm->memslots, mem->slot);
  654. base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
  655. npages = mem->memory_size >> PAGE_SHIFT;
  656. r = -EINVAL;
  657. if (npages > KVM_MEM_MAX_NR_PAGES)
  658. goto out;
  659. if (!npages)
  660. mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
  661. new = old = *slot;
  662. new.id = mem->slot;
  663. new.base_gfn = base_gfn;
  664. new.npages = npages;
  665. new.flags = mem->flags;
  666. r = -EINVAL;
  667. if (npages) {
  668. if (!old.npages)
  669. change = KVM_MR_CREATE;
  670. else { /* Modify an existing slot. */
  671. if ((mem->userspace_addr != old.userspace_addr) ||
  672. (npages != old.npages) ||
  673. ((new.flags ^ old.flags) & KVM_MEM_READONLY))
  674. goto out;
  675. if (base_gfn != old.base_gfn)
  676. change = KVM_MR_MOVE;
  677. else if (new.flags != old.flags)
  678. change = KVM_MR_FLAGS_ONLY;
  679. else { /* Nothing to change. */
  680. r = 0;
  681. goto out;
  682. }
  683. }
  684. } else if (old.npages) {
  685. change = KVM_MR_DELETE;
  686. } else /* Modify a non-existent slot: disallowed. */
  687. goto out;
  688. if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
  689. /* Check for overlaps */
  690. r = -EEXIST;
  691. kvm_for_each_memslot(slot, kvm->memslots) {
  692. if ((slot->id >= KVM_USER_MEM_SLOTS) ||
  693. (slot->id == mem->slot))
  694. continue;
  695. if (!((base_gfn + npages <= slot->base_gfn) ||
  696. (base_gfn >= slot->base_gfn + slot->npages)))
  697. goto out;
  698. }
  699. }
  700. /* Free page dirty bitmap if unneeded */
  701. if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
  702. new.dirty_bitmap = NULL;
  703. r = -ENOMEM;
  704. if (change == KVM_MR_CREATE) {
  705. new.userspace_addr = mem->userspace_addr;
  706. if (kvm_arch_create_memslot(&new, npages))
  707. goto out_free;
  708. }
  709. /* Allocate page dirty bitmap if needed */
  710. if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
  711. if (kvm_create_dirty_bitmap(&new) < 0)
  712. goto out_free;
  713. }
  714. if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
  715. r = -ENOMEM;
  716. slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
  717. GFP_KERNEL);
  718. if (!slots)
  719. goto out_free;
  720. slot = id_to_memslot(slots, mem->slot);
  721. slot->flags |= KVM_MEMSLOT_INVALID;
  722. old_memslots = install_new_memslots(kvm, slots, NULL);
  723. /* slot was deleted or moved, clear iommu mapping */
  724. kvm_iommu_unmap_pages(kvm, &old);
  725. /* From this point no new shadow pages pointing to a deleted,
  726. * or moved, memslot will be created.
  727. *
  728. * validation of sp->gfn happens in:
  729. * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
  730. * - kvm_is_visible_gfn (mmu_check_roots)
  731. */
  732. kvm_arch_flush_shadow_memslot(kvm, slot);
  733. slots = old_memslots;
  734. }
  735. r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
  736. if (r)
  737. goto out_slots;
  738. r = -ENOMEM;
  739. /*
  740. * We can re-use the old_memslots from above, the only difference
  741. * from the currently installed memslots is the invalid flag. This
  742. * will get overwritten by update_memslots anyway.
  743. */
  744. if (!slots) {
  745. slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
  746. GFP_KERNEL);
  747. if (!slots)
  748. goto out_free;
  749. }
  750. /*
  751. * IOMMU mapping: New slots need to be mapped. Old slots need to be
  752. * un-mapped and re-mapped if their base changes. Since base change
  753. * unmapping is handled above with slot deletion, mapping alone is
  754. * needed here. Anything else the iommu might care about for existing
  755. * slots (size changes, userspace addr changes and read-only flag
  756. * changes) is disallowed above, so any other attribute changes getting
  757. * here can be skipped.
  758. */
  759. if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
  760. r = kvm_iommu_map_pages(kvm, &new);
  761. if (r)
  762. goto out_slots;
  763. }
  764. /* actual memory is freed via old in kvm_free_physmem_slot below */
  765. if (change == KVM_MR_DELETE) {
  766. new.dirty_bitmap = NULL;
  767. memset(&new.arch, 0, sizeof(new.arch));
  768. }
  769. old_memslots = install_new_memslots(kvm, slots, &new);
  770. kvm_arch_commit_memory_region(kvm, mem, &old, change);
  771. kvm_free_physmem_slot(&old, &new);
  772. kfree(old_memslots);
  773. return 0;
  774. out_slots:
  775. kfree(slots);
  776. out_free:
  777. kvm_free_physmem_slot(&new, &old);
  778. out:
  779. return r;
  780. }
  781. EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
  782. int kvm_set_memory_region(struct kvm *kvm,
  783. struct kvm_userspace_memory_region *mem)
  784. {
  785. int r;
  786. mutex_lock(&kvm->slots_lock);
  787. r = __kvm_set_memory_region(kvm, mem);
  788. mutex_unlock(&kvm->slots_lock);
  789. return r;
  790. }
  791. EXPORT_SYMBOL_GPL(kvm_set_memory_region);
  792. int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  793. struct kvm_userspace_memory_region *mem)
  794. {
  795. if (mem->slot >= KVM_USER_MEM_SLOTS)
  796. return -EINVAL;
  797. return kvm_set_memory_region(kvm, mem);
  798. }
  799. int kvm_get_dirty_log(struct kvm *kvm,
  800. struct kvm_dirty_log *log, int *is_dirty)
  801. {
  802. struct kvm_memory_slot *memslot;
  803. int r, i;
  804. unsigned long n;
  805. unsigned long any = 0;
  806. r = -EINVAL;
  807. if (log->slot >= KVM_USER_MEM_SLOTS)
  808. goto out;
  809. memslot = id_to_memslot(kvm->memslots, log->slot);
  810. r = -ENOENT;
  811. if (!memslot->dirty_bitmap)
  812. goto out;
  813. n = kvm_dirty_bitmap_bytes(memslot);
  814. for (i = 0; !any && i < n/sizeof(long); ++i)
  815. any = memslot->dirty_bitmap[i];
  816. r = -EFAULT;
  817. if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
  818. goto out;
  819. if (any)
  820. *is_dirty = 1;
  821. r = 0;
  822. out:
  823. return r;
  824. }
  825. bool kvm_largepages_enabled(void)
  826. {
  827. return largepages_enabled;
  828. }
  829. void kvm_disable_largepages(void)
  830. {
  831. largepages_enabled = false;
  832. }
  833. EXPORT_SYMBOL_GPL(kvm_disable_largepages);
  834. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  835. {
  836. return __gfn_to_memslot(kvm_memslots(kvm), gfn);
  837. }
  838. EXPORT_SYMBOL_GPL(gfn_to_memslot);
  839. int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
  840. {
  841. struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
  842. if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
  843. memslot->flags & KVM_MEMSLOT_INVALID)
  844. return 0;
  845. return 1;
  846. }
  847. EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
  848. unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
  849. {
  850. struct vm_area_struct *vma;
  851. unsigned long addr, size;
  852. size = PAGE_SIZE;
  853. addr = gfn_to_hva(kvm, gfn);
  854. if (kvm_is_error_hva(addr))
  855. return PAGE_SIZE;
  856. down_read(&current->mm->mmap_sem);
  857. vma = find_vma(current->mm, addr);
  858. if (!vma)
  859. goto out;
  860. size = vma_kernel_pagesize(vma);
  861. out:
  862. up_read(&current->mm->mmap_sem);
  863. return size;
  864. }
  865. static bool memslot_is_readonly(struct kvm_memory_slot *slot)
  866. {
  867. return slot->flags & KVM_MEM_READONLY;
  868. }
  869. static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
  870. gfn_t *nr_pages, bool write)
  871. {
  872. if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
  873. return KVM_HVA_ERR_BAD;
  874. if (memslot_is_readonly(slot) && write)
  875. return KVM_HVA_ERR_RO_BAD;
  876. if (nr_pages)
  877. *nr_pages = slot->npages - (gfn - slot->base_gfn);
  878. return __gfn_to_hva_memslot(slot, gfn);
  879. }
  880. static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
  881. gfn_t *nr_pages)
  882. {
  883. return __gfn_to_hva_many(slot, gfn, nr_pages, true);
  884. }
  885. unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
  886. gfn_t gfn)
  887. {
  888. return gfn_to_hva_many(slot, gfn, NULL);
  889. }
  890. EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
  891. unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
  892. {
  893. return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
  894. }
  895. EXPORT_SYMBOL_GPL(gfn_to_hva);
  896. /*
  897. * The hva returned by this function is only allowed to be read.
  898. * It should pair with kvm_read_hva() or kvm_read_hva_atomic().
  899. */
  900. static unsigned long gfn_to_hva_read(struct kvm *kvm, gfn_t gfn)
  901. {
  902. return __gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL, false);
  903. }
  904. static int kvm_read_hva(void *data, void __user *hva, int len)
  905. {
  906. return __copy_from_user(data, hva, len);
  907. }
  908. static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
  909. {
  910. return __copy_from_user_inatomic(data, hva, len);
  911. }
  912. static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
  913. unsigned long start, int write, struct page **page)
  914. {
  915. int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
  916. if (write)
  917. flags |= FOLL_WRITE;
  918. return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
  919. }
  920. static inline int check_user_page_hwpoison(unsigned long addr)
  921. {
  922. int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
  923. rc = __get_user_pages(current, current->mm, addr, 1,
  924. flags, NULL, NULL, NULL);
  925. return rc == -EHWPOISON;
  926. }
  927. /*
  928. * The atomic path to get the writable pfn which will be stored in @pfn,
  929. * true indicates success, otherwise false is returned.
  930. */
  931. static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
  932. bool write_fault, bool *writable, pfn_t *pfn)
  933. {
  934. struct page *page[1];
  935. int npages;
  936. if (!(async || atomic))
  937. return false;
  938. /*
  939. * Fast pin a writable pfn only if it is a write fault request
  940. * or the caller allows to map a writable pfn for a read fault
  941. * request.
  942. */
  943. if (!(write_fault || writable))
  944. return false;
  945. npages = __get_user_pages_fast(addr, 1, 1, page);
  946. if (npages == 1) {
  947. *pfn = page_to_pfn(page[0]);
  948. if (writable)
  949. *writable = true;
  950. return true;
  951. }
  952. return false;
  953. }
  954. /*
  955. * The slow path to get the pfn of the specified host virtual address,
  956. * 1 indicates success, -errno is returned if error is detected.
  957. */
  958. static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
  959. bool *writable, pfn_t *pfn)
  960. {
  961. struct page *page[1];
  962. int npages = 0;
  963. might_sleep();
  964. if (writable)
  965. *writable = write_fault;
  966. if (async) {
  967. down_read(&current->mm->mmap_sem);
  968. npages = get_user_page_nowait(current, current->mm,
  969. addr, write_fault, page);
  970. up_read(&current->mm->mmap_sem);
  971. } else
  972. npages = get_user_pages_fast(addr, 1, write_fault,
  973. page);
  974. if (npages != 1)
  975. return npages;
  976. /* map read fault as writable if possible */
  977. if (unlikely(!write_fault) && writable) {
  978. struct page *wpage[1];
  979. npages = __get_user_pages_fast(addr, 1, 1, wpage);
  980. if (npages == 1) {
  981. *writable = true;
  982. put_page(page[0]);
  983. page[0] = wpage[0];
  984. }
  985. npages = 1;
  986. }
  987. *pfn = page_to_pfn(page[0]);
  988. return npages;
  989. }
  990. static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
  991. {
  992. if (unlikely(!(vma->vm_flags & VM_READ)))
  993. return false;
  994. if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
  995. return false;
  996. return true;
  997. }
  998. /*
  999. * Pin guest page in memory and return its pfn.
  1000. * @addr: host virtual address which maps memory to the guest
  1001. * @atomic: whether this function can sleep
  1002. * @async: whether this function need to wait IO complete if the
  1003. * host page is not in the memory
  1004. * @write_fault: whether we should get a writable host page
  1005. * @writable: whether it allows to map a writable host page for !@write_fault
  1006. *
  1007. * The function will map a writable host page for these two cases:
  1008. * 1): @write_fault = true
  1009. * 2): @write_fault = false && @writable, @writable will tell the caller
  1010. * whether the mapping is writable.
  1011. */
  1012. static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
  1013. bool write_fault, bool *writable)
  1014. {
  1015. struct vm_area_struct *vma;
  1016. pfn_t pfn = 0;
  1017. int npages;
  1018. /* we can do it either atomically or asynchronously, not both */
  1019. BUG_ON(atomic && async);
  1020. if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
  1021. return pfn;
  1022. if (atomic)
  1023. return KVM_PFN_ERR_FAULT;
  1024. npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
  1025. if (npages == 1)
  1026. return pfn;
  1027. down_read(&current->mm->mmap_sem);
  1028. if (npages == -EHWPOISON ||
  1029. (!async && check_user_page_hwpoison(addr))) {
  1030. pfn = KVM_PFN_ERR_HWPOISON;
  1031. goto exit;
  1032. }
  1033. vma = find_vma_intersection(current->mm, addr, addr + 1);
  1034. if (vma == NULL)
  1035. pfn = KVM_PFN_ERR_FAULT;
  1036. else if ((vma->vm_flags & VM_PFNMAP)) {
  1037. pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
  1038. vma->vm_pgoff;
  1039. BUG_ON(!kvm_is_mmio_pfn(pfn));
  1040. } else {
  1041. if (async && vma_is_valid(vma, write_fault))
  1042. *async = true;
  1043. pfn = KVM_PFN_ERR_FAULT;
  1044. }
  1045. exit:
  1046. up_read(&current->mm->mmap_sem);
  1047. return pfn;
  1048. }
  1049. static pfn_t
  1050. __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
  1051. bool *async, bool write_fault, bool *writable)
  1052. {
  1053. unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
  1054. if (addr == KVM_HVA_ERR_RO_BAD)
  1055. return KVM_PFN_ERR_RO_FAULT;
  1056. if (kvm_is_error_hva(addr))
  1057. return KVM_PFN_NOSLOT;
  1058. /* Do not map writable pfn in the readonly memslot. */
  1059. if (writable && memslot_is_readonly(slot)) {
  1060. *writable = false;
  1061. writable = NULL;
  1062. }
  1063. return hva_to_pfn(addr, atomic, async, write_fault,
  1064. writable);
  1065. }
  1066. static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
  1067. bool write_fault, bool *writable)
  1068. {
  1069. struct kvm_memory_slot *slot;
  1070. if (async)
  1071. *async = false;
  1072. slot = gfn_to_memslot(kvm, gfn);
  1073. return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
  1074. writable);
  1075. }
  1076. pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
  1077. {
  1078. return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
  1079. }
  1080. EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
  1081. pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
  1082. bool write_fault, bool *writable)
  1083. {
  1084. return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
  1085. }
  1086. EXPORT_SYMBOL_GPL(gfn_to_pfn_async);
  1087. pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
  1088. {
  1089. return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
  1090. }
  1091. EXPORT_SYMBOL_GPL(gfn_to_pfn);
  1092. pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
  1093. bool *writable)
  1094. {
  1095. return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
  1096. }
  1097. EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
  1098. pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
  1099. {
  1100. return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
  1101. }
  1102. pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
  1103. {
  1104. return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
  1105. }
  1106. EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
  1107. int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
  1108. int nr_pages)
  1109. {
  1110. unsigned long addr;
  1111. gfn_t entry;
  1112. addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
  1113. if (kvm_is_error_hva(addr))
  1114. return -1;
  1115. if (entry < nr_pages)
  1116. return 0;
  1117. return __get_user_pages_fast(addr, nr_pages, 1, pages);
  1118. }
  1119. EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
  1120. static struct page *kvm_pfn_to_page(pfn_t pfn)
  1121. {
  1122. if (is_error_noslot_pfn(pfn))
  1123. return KVM_ERR_PTR_BAD_PAGE;
  1124. if (kvm_is_mmio_pfn(pfn)) {
  1125. WARN_ON(1);
  1126. return KVM_ERR_PTR_BAD_PAGE;
  1127. }
  1128. return pfn_to_page(pfn);
  1129. }
  1130. struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
  1131. {
  1132. pfn_t pfn;
  1133. pfn = gfn_to_pfn(kvm, gfn);
  1134. return kvm_pfn_to_page(pfn);
  1135. }
  1136. EXPORT_SYMBOL_GPL(gfn_to_page);
  1137. void kvm_release_page_clean(struct page *page)
  1138. {
  1139. WARN_ON(is_error_page(page));
  1140. kvm_release_pfn_clean(page_to_pfn(page));
  1141. }
  1142. EXPORT_SYMBOL_GPL(kvm_release_page_clean);
  1143. void kvm_release_pfn_clean(pfn_t pfn)
  1144. {
  1145. if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn))
  1146. put_page(pfn_to_page(pfn));
  1147. }
  1148. EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
  1149. void kvm_release_page_dirty(struct page *page)
  1150. {
  1151. WARN_ON(is_error_page(page));
  1152. kvm_release_pfn_dirty(page_to_pfn(page));
  1153. }
  1154. EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
  1155. void kvm_release_pfn_dirty(pfn_t pfn)
  1156. {
  1157. kvm_set_pfn_dirty(pfn);
  1158. kvm_release_pfn_clean(pfn);
  1159. }
  1160. EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
  1161. void kvm_set_page_dirty(struct page *page)
  1162. {
  1163. kvm_set_pfn_dirty(page_to_pfn(page));
  1164. }
  1165. EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
  1166. void kvm_set_pfn_dirty(pfn_t pfn)
  1167. {
  1168. if (!kvm_is_mmio_pfn(pfn)) {
  1169. struct page *page = pfn_to_page(pfn);
  1170. if (!PageReserved(page))
  1171. SetPageDirty(page);
  1172. }
  1173. }
  1174. EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
  1175. void kvm_set_pfn_accessed(pfn_t pfn)
  1176. {
  1177. if (!kvm_is_mmio_pfn(pfn))
  1178. mark_page_accessed(pfn_to_page(pfn));
  1179. }
  1180. EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
  1181. void kvm_get_pfn(pfn_t pfn)
  1182. {
  1183. if (!kvm_is_mmio_pfn(pfn))
  1184. get_page(pfn_to_page(pfn));
  1185. }
  1186. EXPORT_SYMBOL_GPL(kvm_get_pfn);
  1187. static int next_segment(unsigned long len, int offset)
  1188. {
  1189. if (len > PAGE_SIZE - offset)
  1190. return PAGE_SIZE - offset;
  1191. else
  1192. return len;
  1193. }
  1194. int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
  1195. int len)
  1196. {
  1197. int r;
  1198. unsigned long addr;
  1199. addr = gfn_to_hva_read(kvm, gfn);
  1200. if (kvm_is_error_hva(addr))
  1201. return -EFAULT;
  1202. r = kvm_read_hva(data, (void __user *)addr + offset, len);
  1203. if (r)
  1204. return -EFAULT;
  1205. return 0;
  1206. }
  1207. EXPORT_SYMBOL_GPL(kvm_read_guest_page);
  1208. int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
  1209. {
  1210. gfn_t gfn = gpa >> PAGE_SHIFT;
  1211. int seg;
  1212. int offset = offset_in_page(gpa);
  1213. int ret;
  1214. while ((seg = next_segment(len, offset)) != 0) {
  1215. ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
  1216. if (ret < 0)
  1217. return ret;
  1218. offset = 0;
  1219. len -= seg;
  1220. data += seg;
  1221. ++gfn;
  1222. }
  1223. return 0;
  1224. }
  1225. EXPORT_SYMBOL_GPL(kvm_read_guest);
  1226. int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
  1227. unsigned long len)
  1228. {
  1229. int r;
  1230. unsigned long addr;
  1231. gfn_t gfn = gpa >> PAGE_SHIFT;
  1232. int offset = offset_in_page(gpa);
  1233. addr = gfn_to_hva_read(kvm, gfn);
  1234. if (kvm_is_error_hva(addr))
  1235. return -EFAULT;
  1236. pagefault_disable();
  1237. r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
  1238. pagefault_enable();
  1239. if (r)
  1240. return -EFAULT;
  1241. return 0;
  1242. }
  1243. EXPORT_SYMBOL(kvm_read_guest_atomic);
  1244. int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
  1245. int offset, int len)
  1246. {
  1247. int r;
  1248. unsigned long addr;
  1249. addr = gfn_to_hva(kvm, gfn);
  1250. if (kvm_is_error_hva(addr))
  1251. return -EFAULT;
  1252. r = __copy_to_user((void __user *)addr + offset, data, len);
  1253. if (r)
  1254. return -EFAULT;
  1255. mark_page_dirty(kvm, gfn);
  1256. return 0;
  1257. }
  1258. EXPORT_SYMBOL_GPL(kvm_write_guest_page);
  1259. int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
  1260. unsigned long len)
  1261. {
  1262. gfn_t gfn = gpa >> PAGE_SHIFT;
  1263. int seg;
  1264. int offset = offset_in_page(gpa);
  1265. int ret;
  1266. while ((seg = next_segment(len, offset)) != 0) {
  1267. ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
  1268. if (ret < 0)
  1269. return ret;
  1270. offset = 0;
  1271. len -= seg;
  1272. data += seg;
  1273. ++gfn;
  1274. }
  1275. return 0;
  1276. }
  1277. int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1278. gpa_t gpa)
  1279. {
  1280. struct kvm_memslots *slots = kvm_memslots(kvm);
  1281. int offset = offset_in_page(gpa);
  1282. gfn_t gfn = gpa >> PAGE_SHIFT;
  1283. ghc->gpa = gpa;
  1284. ghc->generation = slots->generation;
  1285. ghc->memslot = gfn_to_memslot(kvm, gfn);
  1286. ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL);
  1287. if (!kvm_is_error_hva(ghc->hva))
  1288. ghc->hva += offset;
  1289. else
  1290. return -EFAULT;
  1291. return 0;
  1292. }
  1293. EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
  1294. int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1295. void *data, unsigned long len)
  1296. {
  1297. struct kvm_memslots *slots = kvm_memslots(kvm);
  1298. int r;
  1299. if (slots->generation != ghc->generation)
  1300. kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
  1301. if (kvm_is_error_hva(ghc->hva))
  1302. return -EFAULT;
  1303. r = __copy_to_user((void __user *)ghc->hva, data, len);
  1304. if (r)
  1305. return -EFAULT;
  1306. mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);
  1307. return 0;
  1308. }
  1309. EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
  1310. int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1311. void *data, unsigned long len)
  1312. {
  1313. struct kvm_memslots *slots = kvm_memslots(kvm);
  1314. int r;
  1315. if (slots->generation != ghc->generation)
  1316. kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
  1317. if (kvm_is_error_hva(ghc->hva))
  1318. return -EFAULT;
  1319. r = __copy_from_user(data, (void __user *)ghc->hva, len);
  1320. if (r)
  1321. return -EFAULT;
  1322. return 0;
  1323. }
  1324. EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
  1325. int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
  1326. {
  1327. return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
  1328. offset, len);
  1329. }
  1330. EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
  1331. int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
  1332. {
  1333. gfn_t gfn = gpa >> PAGE_SHIFT;
  1334. int seg;
  1335. int offset = offset_in_page(gpa);
  1336. int ret;
  1337. while ((seg = next_segment(len, offset)) != 0) {
  1338. ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
  1339. if (ret < 0)
  1340. return ret;
  1341. offset = 0;
  1342. len -= seg;
  1343. ++gfn;
  1344. }
  1345. return 0;
  1346. }
  1347. EXPORT_SYMBOL_GPL(kvm_clear_guest);
  1348. void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
  1349. gfn_t gfn)
  1350. {
  1351. if (memslot && memslot->dirty_bitmap) {
  1352. unsigned long rel_gfn = gfn - memslot->base_gfn;
  1353. set_bit_le(rel_gfn, memslot->dirty_bitmap);
  1354. }
  1355. }
  1356. void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
  1357. {
  1358. struct kvm_memory_slot *memslot;
  1359. memslot = gfn_to_memslot(kvm, gfn);
  1360. mark_page_dirty_in_slot(kvm, memslot, gfn);
  1361. }
  1362. /*
  1363. * The vCPU has executed a HLT instruction with in-kernel mode enabled.
  1364. */
  1365. void kvm_vcpu_block(struct kvm_vcpu *vcpu)
  1366. {
  1367. DEFINE_WAIT(wait);
  1368. for (;;) {
  1369. prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
  1370. if (kvm_arch_vcpu_runnable(vcpu)) {
  1371. kvm_make_request(KVM_REQ_UNHALT, vcpu);
  1372. break;
  1373. }
  1374. if (kvm_cpu_has_pending_timer(vcpu))
  1375. break;
  1376. if (signal_pending(current))
  1377. break;
  1378. schedule();
  1379. }
  1380. finish_wait(&vcpu->wq, &wait);
  1381. }
  1382. #ifndef CONFIG_S390
  1383. /*
  1384. * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
  1385. */
  1386. void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
  1387. {
  1388. int me;
  1389. int cpu = vcpu->cpu;
  1390. wait_queue_head_t *wqp;
  1391. wqp = kvm_arch_vcpu_wq(vcpu);
  1392. if (waitqueue_active(wqp)) {
  1393. wake_up_interruptible(wqp);
  1394. ++vcpu->stat.halt_wakeup;
  1395. }
  1396. me = get_cpu();
  1397. if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
  1398. if (kvm_arch_vcpu_should_kick(vcpu))
  1399. smp_send_reschedule(cpu);
  1400. put_cpu();
  1401. }
  1402. EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
  1403. #endif /* !CONFIG_S390 */
  1404. void kvm_resched(struct kvm_vcpu *vcpu)
  1405. {
  1406. if (!need_resched())
  1407. return;
  1408. cond_resched();
  1409. }
  1410. EXPORT_SYMBOL_GPL(kvm_resched);
  1411. bool kvm_vcpu_yield_to(struct kvm_vcpu *target)
  1412. {
  1413. struct pid *pid;
  1414. struct task_struct *task = NULL;
  1415. bool ret = false;
  1416. rcu_read_lock();
  1417. pid = rcu_dereference(target->pid);
  1418. if (pid)
  1419. task = get_pid_task(target->pid, PIDTYPE_PID);
  1420. rcu_read_unlock();
  1421. if (!task)
  1422. return ret;
  1423. if (task->flags & PF_VCPU) {
  1424. put_task_struct(task);
  1425. return ret;
  1426. }
  1427. ret = yield_to(task, 1);
  1428. put_task_struct(task);
  1429. return ret;
  1430. }
  1431. EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
  1432. #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
  1433. /*
  1434. * Helper that checks whether a VCPU is eligible for directed yield.
  1435. * Most eligible candidate to yield is decided by following heuristics:
  1436. *
  1437. * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
  1438. * (preempted lock holder), indicated by @in_spin_loop.
  1439. * Set at the beiginning and cleared at the end of interception/PLE handler.
  1440. *
  1441. * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
  1442. * chance last time (mostly it has become eligible now since we have probably
  1443. * yielded to lockholder in last iteration. This is done by toggling
  1444. * @dy_eligible each time a VCPU checked for eligibility.)
  1445. *
  1446. * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
  1447. * to preempted lock-holder could result in wrong VCPU selection and CPU
  1448. * burning. Giving priority for a potential lock-holder increases lock
  1449. * progress.
  1450. *
  1451. * Since algorithm is based on heuristics, accessing another VCPU data without
  1452. * locking does not harm. It may result in trying to yield to same VCPU, fail
  1453. * and continue with next VCPU and so on.
  1454. */
  1455. bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
  1456. {
  1457. bool eligible;
  1458. eligible = !vcpu->spin_loop.in_spin_loop ||
  1459. (vcpu->spin_loop.in_spin_loop &&
  1460. vcpu->spin_loop.dy_eligible);
  1461. if (vcpu->spin_loop.in_spin_loop)
  1462. kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
  1463. return eligible;
  1464. }
  1465. #endif
  1466. void kvm_vcpu_on_spin(struct kvm_vcpu *me)
  1467. {
  1468. struct kvm *kvm = me->kvm;
  1469. struct kvm_vcpu *vcpu;
  1470. int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
  1471. int yielded = 0;
  1472. int try = 3;
  1473. int pass;
  1474. int i;
  1475. kvm_vcpu_set_in_spin_loop(me, true);
  1476. /*
  1477. * We boost the priority of a VCPU that is runnable but not
  1478. * currently running, because it got preempted by something
  1479. * else and called schedule in __vcpu_run. Hopefully that
  1480. * VCPU is holding the lock that we need and will release it.
  1481. * We approximate round-robin by starting at the last boosted VCPU.
  1482. */
  1483. for (pass = 0; pass < 2 && !yielded && try; pass++) {
  1484. kvm_for_each_vcpu(i, vcpu, kvm) {
  1485. if (!pass && i <= last_boosted_vcpu) {
  1486. i = last_boosted_vcpu;
  1487. continue;
  1488. } else if (pass && i > last_boosted_vcpu)
  1489. break;
  1490. if (!ACCESS_ONCE(vcpu->preempted))
  1491. continue;
  1492. if (vcpu == me)
  1493. continue;
  1494. if (waitqueue_active(&vcpu->wq))
  1495. continue;
  1496. if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
  1497. continue;
  1498. yielded = kvm_vcpu_yield_to(vcpu);
  1499. if (yielded > 0) {
  1500. kvm->last_boosted_vcpu = i;
  1501. break;
  1502. } else if (yielded < 0) {
  1503. try--;
  1504. if (!try)
  1505. break;
  1506. }
  1507. }
  1508. }
  1509. kvm_vcpu_set_in_spin_loop(me, false);
  1510. /* Ensure vcpu is not eligible during next spinloop */
  1511. kvm_vcpu_set_dy_eligible(me, false);
  1512. }
  1513. EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
  1514. static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1515. {
  1516. struct kvm_vcpu *vcpu = vma->vm_file->private_data;
  1517. struct page *page;
  1518. if (vmf->pgoff == 0)
  1519. page = virt_to_page(vcpu->run);
  1520. #ifdef CONFIG_X86
  1521. else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
  1522. page = virt_to_page(vcpu->arch.pio_data);
  1523. #endif
  1524. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  1525. else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
  1526. page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
  1527. #endif
  1528. else
  1529. return kvm_arch_vcpu_fault(vcpu, vmf);
  1530. get_page(page);
  1531. vmf->page = page;
  1532. return 0;
  1533. }
  1534. static const struct vm_operations_struct kvm_vcpu_vm_ops = {
  1535. .fault = kvm_vcpu_fault,
  1536. };
  1537. static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
  1538. {
  1539. vma->vm_ops = &kvm_vcpu_vm_ops;
  1540. return 0;
  1541. }
  1542. static int kvm_vcpu_release(struct inode *inode, struct file *filp)
  1543. {
  1544. struct kvm_vcpu *vcpu = filp->private_data;
  1545. kvm_put_kvm(vcpu->kvm);
  1546. return 0;
  1547. }
  1548. static struct file_operations kvm_vcpu_fops = {
  1549. .release = kvm_vcpu_release,
  1550. .unlocked_ioctl = kvm_vcpu_ioctl,
  1551. #ifdef CONFIG_COMPAT
  1552. .compat_ioctl = kvm_vcpu_compat_ioctl,
  1553. #endif
  1554. .mmap = kvm_vcpu_mmap,
  1555. .llseek = noop_llseek,
  1556. };
  1557. /*
  1558. * Allocates an inode for the vcpu.
  1559. */
  1560. static int create_vcpu_fd(struct kvm_vcpu *vcpu)
  1561. {
  1562. return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
  1563. }
  1564. /*
  1565. * Creates some virtual cpus. Good luck creating more than one.
  1566. */
  1567. static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
  1568. {
  1569. int r;
  1570. struct kvm_vcpu *vcpu, *v;
  1571. vcpu = kvm_arch_vcpu_create(kvm, id);
  1572. if (IS_ERR(vcpu))
  1573. return PTR_ERR(vcpu);
  1574. preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
  1575. r = kvm_arch_vcpu_setup(vcpu);
  1576. if (r)
  1577. goto vcpu_destroy;
  1578. mutex_lock(&kvm->lock);
  1579. if (!kvm_vcpu_compatible(vcpu)) {
  1580. r = -EINVAL;
  1581. goto unlock_vcpu_destroy;
  1582. }
  1583. if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
  1584. r = -EINVAL;
  1585. goto unlock_vcpu_destroy;
  1586. }
  1587. kvm_for_each_vcpu(r, v, kvm)
  1588. if (v->vcpu_id == id) {
  1589. r = -EEXIST;
  1590. goto unlock_vcpu_destroy;
  1591. }
  1592. BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
  1593. /* Now it's all set up, let userspace reach it */
  1594. kvm_get_kvm(kvm);
  1595. r = create_vcpu_fd(vcpu);
  1596. if (r < 0) {
  1597. kvm_put_kvm(kvm);
  1598. goto unlock_vcpu_destroy;
  1599. }
  1600. kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
  1601. smp_wmb();
  1602. atomic_inc(&kvm->online_vcpus);
  1603. mutex_unlock(&kvm->lock);
  1604. kvm_arch_vcpu_postcreate(vcpu);
  1605. return r;
  1606. unlock_vcpu_destroy:
  1607. mutex_unlock(&kvm->lock);
  1608. vcpu_destroy:
  1609. kvm_arch_vcpu_destroy(vcpu);
  1610. return r;
  1611. }
  1612. static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
  1613. {
  1614. if (sigset) {
  1615. sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
  1616. vcpu->sigset_active = 1;
  1617. vcpu->sigset = *sigset;
  1618. } else
  1619. vcpu->sigset_active = 0;
  1620. return 0;
  1621. }
  1622. static long kvm_vcpu_ioctl(struct file *filp,
  1623. unsigned int ioctl, unsigned long arg)
  1624. {
  1625. struct kvm_vcpu *vcpu = filp->private_data;
  1626. void __user *argp = (void __user *)arg;
  1627. int r;
  1628. struct kvm_fpu *fpu = NULL;
  1629. struct kvm_sregs *kvm_sregs = NULL;
  1630. if (vcpu->kvm->mm != current->mm)
  1631. return -EIO;
  1632. #if defined(CONFIG_S390) || defined(CONFIG_PPC)
  1633. /*
  1634. * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
  1635. * so vcpu_load() would break it.
  1636. */
  1637. if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
  1638. return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  1639. #endif
  1640. r = vcpu_load(vcpu);
  1641. if (r)
  1642. return r;
  1643. switch (ioctl) {
  1644. case KVM_RUN:
  1645. r = -EINVAL;
  1646. if (arg)
  1647. goto out;
  1648. r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
  1649. trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
  1650. break;
  1651. case KVM_GET_REGS: {
  1652. struct kvm_regs *kvm_regs;
  1653. r = -ENOMEM;
  1654. kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
  1655. if (!kvm_regs)
  1656. goto out;
  1657. r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
  1658. if (r)
  1659. goto out_free1;
  1660. r = -EFAULT;
  1661. if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
  1662. goto out_free1;
  1663. r = 0;
  1664. out_free1:
  1665. kfree(kvm_regs);
  1666. break;
  1667. }
  1668. case KVM_SET_REGS: {
  1669. struct kvm_regs *kvm_regs;
  1670. r = -ENOMEM;
  1671. kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
  1672. if (IS_ERR(kvm_regs)) {
  1673. r = PTR_ERR(kvm_regs);
  1674. goto out;
  1675. }
  1676. r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
  1677. kfree(kvm_regs);
  1678. break;
  1679. }
  1680. case KVM_GET_SREGS: {
  1681. kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
  1682. r = -ENOMEM;
  1683. if (!kvm_sregs)
  1684. goto out;
  1685. r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
  1686. if (r)
  1687. goto out;
  1688. r = -EFAULT;
  1689. if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
  1690. goto out;
  1691. r = 0;
  1692. break;
  1693. }
  1694. case KVM_SET_SREGS: {
  1695. kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
  1696. if (IS_ERR(kvm_sregs)) {
  1697. r = PTR_ERR(kvm_sregs);
  1698. kvm_sregs = NULL;
  1699. goto out;
  1700. }
  1701. r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
  1702. break;
  1703. }
  1704. case KVM_GET_MP_STATE: {
  1705. struct kvm_mp_state mp_state;
  1706. r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
  1707. if (r)
  1708. goto out;
  1709. r = -EFAULT;
  1710. if (copy_to_user(argp, &mp_state, sizeof mp_state))
  1711. goto out;
  1712. r = 0;
  1713. break;
  1714. }
  1715. case KVM_SET_MP_STATE: {
  1716. struct kvm_mp_state mp_state;
  1717. r = -EFAULT;
  1718. if (copy_from_user(&mp_state, argp, sizeof mp_state))
  1719. goto out;
  1720. r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
  1721. break;
  1722. }
  1723. case KVM_TRANSLATE: {
  1724. struct kvm_translation tr;
  1725. r = -EFAULT;
  1726. if (copy_from_user(&tr, argp, sizeof tr))
  1727. goto out;
  1728. r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
  1729. if (r)
  1730. goto out;
  1731. r = -EFAULT;
  1732. if (copy_to_user(argp, &tr, sizeof tr))
  1733. goto out;
  1734. r = 0;
  1735. break;
  1736. }
  1737. case KVM_SET_GUEST_DEBUG: {
  1738. struct kvm_guest_debug dbg;
  1739. r = -EFAULT;
  1740. if (copy_from_user(&dbg, argp, sizeof dbg))
  1741. goto out;
  1742. r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
  1743. break;
  1744. }
  1745. case KVM_SET_SIGNAL_MASK: {
  1746. struct kvm_signal_mask __user *sigmask_arg = argp;
  1747. struct kvm_signal_mask kvm_sigmask;
  1748. sigset_t sigset, *p;
  1749. p = NULL;
  1750. if (argp) {
  1751. r = -EFAULT;
  1752. if (copy_from_user(&kvm_sigmask, argp,
  1753. sizeof kvm_sigmask))
  1754. goto out;
  1755. r = -EINVAL;
  1756. if (kvm_sigmask.len != sizeof sigset)
  1757. goto out;
  1758. r = -EFAULT;
  1759. if (copy_from_user(&sigset, sigmask_arg->sigset,
  1760. sizeof sigset))
  1761. goto out;
  1762. p = &sigset;
  1763. }
  1764. r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
  1765. break;
  1766. }
  1767. case KVM_GET_FPU: {
  1768. fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
  1769. r = -ENOMEM;
  1770. if (!fpu)
  1771. goto out;
  1772. r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
  1773. if (r)
  1774. goto out;
  1775. r = -EFAULT;
  1776. if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
  1777. goto out;
  1778. r = 0;
  1779. break;
  1780. }
  1781. case KVM_SET_FPU: {
  1782. fpu = memdup_user(argp, sizeof(*fpu));
  1783. if (IS_ERR(fpu)) {
  1784. r = PTR_ERR(fpu);
  1785. fpu = NULL;
  1786. goto out;
  1787. }
  1788. r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
  1789. break;
  1790. }
  1791. default:
  1792. r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  1793. }
  1794. out:
  1795. vcpu_put(vcpu);
  1796. kfree(fpu);
  1797. kfree(kvm_sregs);
  1798. return r;
  1799. }
  1800. #ifdef CONFIG_COMPAT
  1801. static long kvm_vcpu_compat_ioctl(struct file *filp,
  1802. unsigned int ioctl, unsigned long arg)
  1803. {
  1804. struct kvm_vcpu *vcpu = filp->private_data;
  1805. void __user *argp = compat_ptr(arg);
  1806. int r;
  1807. if (vcpu->kvm->mm != current->mm)
  1808. return -EIO;
  1809. switch (ioctl) {
  1810. case KVM_SET_SIGNAL_MASK: {
  1811. struct kvm_signal_mask __user *sigmask_arg = argp;
  1812. struct kvm_signal_mask kvm_sigmask;
  1813. compat_sigset_t csigset;
  1814. sigset_t sigset;
  1815. if (argp) {
  1816. r = -EFAULT;
  1817. if (copy_from_user(&kvm_sigmask, argp,
  1818. sizeof kvm_sigmask))
  1819. goto out;
  1820. r = -EINVAL;
  1821. if (kvm_sigmask.len != sizeof csigset)
  1822. goto out;
  1823. r = -EFAULT;
  1824. if (copy_from_user(&csigset, sigmask_arg->sigset,
  1825. sizeof csigset))
  1826. goto out;
  1827. sigset_from_compat(&sigset, &csigset);
  1828. r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
  1829. } else
  1830. r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
  1831. break;
  1832. }
  1833. default:
  1834. r = kvm_vcpu_ioctl(filp, ioctl, arg);
  1835. }
  1836. out:
  1837. return r;
  1838. }
  1839. #endif
  1840. static int kvm_device_ioctl_attr(struct kvm_device *dev,
  1841. int (*accessor)(struct kvm_device *dev,
  1842. struct kvm_device_attr *attr),
  1843. unsigned long arg)
  1844. {
  1845. struct kvm_device_attr attr;
  1846. if (!accessor)
  1847. return -EPERM;
  1848. if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
  1849. return -EFAULT;
  1850. return accessor(dev, &attr);
  1851. }
  1852. static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
  1853. unsigned long arg)
  1854. {
  1855. struct kvm_device *dev = filp->private_data;
  1856. switch (ioctl) {
  1857. case KVM_SET_DEVICE_ATTR:
  1858. return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
  1859. case KVM_GET_DEVICE_ATTR:
  1860. return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
  1861. case KVM_HAS_DEVICE_ATTR:
  1862. return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
  1863. default:
  1864. if (dev->ops->ioctl)
  1865. return dev->ops->ioctl(dev, ioctl, arg);
  1866. return -ENOTTY;
  1867. }
  1868. }
  1869. static int kvm_device_release(struct inode *inode, struct file *filp)
  1870. {
  1871. struct kvm_device *dev = filp->private_data;
  1872. struct kvm *kvm = dev->kvm;
  1873. kvm_put_kvm(kvm);
  1874. return 0;
  1875. }
  1876. static const struct file_operations kvm_device_fops = {
  1877. .unlocked_ioctl = kvm_device_ioctl,
  1878. .release = kvm_device_release,
  1879. };
  1880. struct kvm_device *kvm_device_from_filp(struct file *filp)
  1881. {
  1882. if (filp->f_op != &kvm_device_fops)
  1883. return NULL;
  1884. return filp->private_data;
  1885. }
  1886. static int kvm_ioctl_create_device(struct kvm *kvm,
  1887. struct kvm_create_device *cd)
  1888. {
  1889. struct kvm_device_ops *ops = NULL;
  1890. struct kvm_device *dev;
  1891. bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
  1892. int ret;
  1893. switch (cd->type) {
  1894. #ifdef CONFIG_KVM_MPIC
  1895. case KVM_DEV_TYPE_FSL_MPIC_20:
  1896. case KVM_DEV_TYPE_FSL_MPIC_42:
  1897. ops = &kvm_mpic_ops;
  1898. break;
  1899. #endif
  1900. default:
  1901. return -ENODEV;
  1902. }
  1903. if (test)
  1904. return 0;
  1905. dev = kzalloc(sizeof(*dev), GFP_KERNEL);
  1906. if (!dev)
  1907. return -ENOMEM;
  1908. dev->ops = ops;
  1909. dev->kvm = kvm;
  1910. ret = ops->create(dev, cd->type);
  1911. if (ret < 0) {
  1912. kfree(dev);
  1913. return ret;
  1914. }
  1915. ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR);
  1916. if (ret < 0) {
  1917. ops->destroy(dev);
  1918. return ret;
  1919. }
  1920. list_add(&dev->vm_node, &kvm->devices);
  1921. kvm_get_kvm(kvm);
  1922. cd->fd = ret;
  1923. return 0;
  1924. }
  1925. static long kvm_vm_ioctl(struct file *filp,
  1926. unsigned int ioctl, unsigned long arg)
  1927. {
  1928. struct kvm *kvm = filp->private_data;
  1929. void __user *argp = (void __user *)arg;
  1930. int r;
  1931. if (kvm->mm != current->mm)
  1932. return -EIO;
  1933. switch (ioctl) {
  1934. case KVM_CREATE_VCPU:
  1935. r = kvm_vm_ioctl_create_vcpu(kvm, arg);
  1936. break;
  1937. case KVM_SET_USER_MEMORY_REGION: {
  1938. struct kvm_userspace_memory_region kvm_userspace_mem;
  1939. r = -EFAULT;
  1940. if (copy_from_user(&kvm_userspace_mem, argp,
  1941. sizeof kvm_userspace_mem))
  1942. goto out;
  1943. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
  1944. break;
  1945. }
  1946. case KVM_GET_DIRTY_LOG: {
  1947. struct kvm_dirty_log log;
  1948. r = -EFAULT;
  1949. if (copy_from_user(&log, argp, sizeof log))
  1950. goto out;
  1951. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  1952. break;
  1953. }
  1954. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  1955. case KVM_REGISTER_COALESCED_MMIO: {
  1956. struct kvm_coalesced_mmio_zone zone;
  1957. r = -EFAULT;
  1958. if (copy_from_user(&zone, argp, sizeof zone))
  1959. goto out;
  1960. r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
  1961. break;
  1962. }
  1963. case KVM_UNREGISTER_COALESCED_MMIO: {
  1964. struct kvm_coalesced_mmio_zone zone;
  1965. r = -EFAULT;
  1966. if (copy_from_user(&zone, argp, sizeof zone))
  1967. goto out;
  1968. r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
  1969. break;
  1970. }
  1971. #endif
  1972. case KVM_IRQFD: {
  1973. struct kvm_irqfd data;
  1974. r = -EFAULT;
  1975. if (copy_from_user(&data, argp, sizeof data))
  1976. goto out;
  1977. r = kvm_irqfd(kvm, &data);
  1978. break;
  1979. }
  1980. case KVM_IOEVENTFD: {
  1981. struct kvm_ioeventfd data;
  1982. r = -EFAULT;
  1983. if (copy_from_user(&data, argp, sizeof data))
  1984. goto out;
  1985. r = kvm_ioeventfd(kvm, &data);
  1986. break;
  1987. }
  1988. #ifdef CONFIG_KVM_APIC_ARCHITECTURE
  1989. case KVM_SET_BOOT_CPU_ID:
  1990. r = 0;
  1991. mutex_lock(&kvm->lock);
  1992. if (atomic_read(&kvm->online_vcpus) != 0)
  1993. r = -EBUSY;
  1994. else
  1995. kvm->bsp_vcpu_id = arg;
  1996. mutex_unlock(&kvm->lock);
  1997. break;
  1998. #endif
  1999. #ifdef CONFIG_HAVE_KVM_MSI
  2000. case KVM_SIGNAL_MSI: {
  2001. struct kvm_msi msi;
  2002. r = -EFAULT;
  2003. if (copy_from_user(&msi, argp, sizeof msi))
  2004. goto out;
  2005. r = kvm_send_userspace_msi(kvm, &msi);
  2006. break;
  2007. }
  2008. #endif
  2009. #ifdef __KVM_HAVE_IRQ_LINE
  2010. case KVM_IRQ_LINE_STATUS:
  2011. case KVM_IRQ_LINE: {
  2012. struct kvm_irq_level irq_event;
  2013. r = -EFAULT;
  2014. if (copy_from_user(&irq_event, argp, sizeof irq_event))
  2015. goto out;
  2016. r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
  2017. ioctl == KVM_IRQ_LINE_STATUS);
  2018. if (r)
  2019. goto out;
  2020. r = -EFAULT;
  2021. if (ioctl == KVM_IRQ_LINE_STATUS) {
  2022. if (copy_to_user(argp, &irq_event, sizeof irq_event))
  2023. goto out;
  2024. }
  2025. r = 0;
  2026. break;
  2027. }
  2028. #endif
  2029. #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
  2030. case KVM_SET_GSI_ROUTING: {
  2031. struct kvm_irq_routing routing;
  2032. struct kvm_irq_routing __user *urouting;
  2033. struct kvm_irq_routing_entry *entries;
  2034. r = -EFAULT;
  2035. if (copy_from_user(&routing, argp, sizeof(routing)))
  2036. goto out;
  2037. r = -EINVAL;
  2038. if (routing.nr >= KVM_MAX_IRQ_ROUTES)
  2039. goto out;
  2040. if (routing.flags)
  2041. goto out;
  2042. r = -ENOMEM;
  2043. entries = vmalloc(routing.nr * sizeof(*entries));
  2044. if (!entries)
  2045. goto out;
  2046. r = -EFAULT;
  2047. urouting = argp;
  2048. if (copy_from_user(entries, urouting->entries,
  2049. routing.nr * sizeof(*entries)))
  2050. goto out_free_irq_routing;
  2051. r = kvm_set_irq_routing(kvm, entries, routing.nr,
  2052. routing.flags);
  2053. out_free_irq_routing:
  2054. vfree(entries);
  2055. break;
  2056. }
  2057. #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
  2058. case KVM_CREATE_DEVICE: {
  2059. struct kvm_create_device cd;
  2060. r = -EFAULT;
  2061. if (copy_from_user(&cd, argp, sizeof(cd)))
  2062. goto out;
  2063. r = kvm_ioctl_create_device(kvm, &cd);
  2064. if (r)
  2065. goto out;
  2066. r = -EFAULT;
  2067. if (copy_to_user(argp, &cd, sizeof(cd)))
  2068. goto out;
  2069. r = 0;
  2070. break;
  2071. }
  2072. default:
  2073. r = kvm_arch_vm_ioctl(filp, ioctl, arg);
  2074. if (r == -ENOTTY)
  2075. r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
  2076. }
  2077. out:
  2078. return r;
  2079. }
  2080. #ifdef CONFIG_COMPAT
  2081. struct compat_kvm_dirty_log {
  2082. __u32 slot;
  2083. __u32 padding1;
  2084. union {
  2085. compat_uptr_t dirty_bitmap; /* one bit per page */
  2086. __u64 padding2;
  2087. };
  2088. };
  2089. static long kvm_vm_compat_ioctl(struct file *filp,
  2090. unsigned int ioctl, unsigned long arg)
  2091. {
  2092. struct kvm *kvm = filp->private_data;
  2093. int r;
  2094. if (kvm->mm != current->mm)
  2095. return -EIO;
  2096. switch (ioctl) {
  2097. case KVM_GET_DIRTY_LOG: {
  2098. struct compat_kvm_dirty_log compat_log;
  2099. struct kvm_dirty_log log;
  2100. r = -EFAULT;
  2101. if (copy_from_user(&compat_log, (void __user *)arg,
  2102. sizeof(compat_log)))
  2103. goto out;
  2104. log.slot = compat_log.slot;
  2105. log.padding1 = compat_log.padding1;
  2106. log.padding2 = compat_log.padding2;
  2107. log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
  2108. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  2109. break;
  2110. }
  2111. default:
  2112. r = kvm_vm_ioctl(filp, ioctl, arg);
  2113. }
  2114. out:
  2115. return r;
  2116. }
  2117. #endif
  2118. static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  2119. {
  2120. struct page *page[1];
  2121. unsigned long addr;
  2122. int npages;
  2123. gfn_t gfn = vmf->pgoff;
  2124. struct kvm *kvm = vma->vm_file->private_data;
  2125. addr = gfn_to_hva(kvm, gfn);
  2126. if (kvm_is_error_hva(addr))
  2127. return VM_FAULT_SIGBUS;
  2128. npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
  2129. NULL);
  2130. if (unlikely(npages != 1))
  2131. return VM_FAULT_SIGBUS;
  2132. vmf->page = page[0];
  2133. return 0;
  2134. }
  2135. static const struct vm_operations_struct kvm_vm_vm_ops = {
  2136. .fault = kvm_vm_fault,
  2137. };
  2138. static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
  2139. {
  2140. vma->vm_ops = &kvm_vm_vm_ops;
  2141. return 0;
  2142. }
  2143. static struct file_operations kvm_vm_fops = {
  2144. .release = kvm_vm_release,
  2145. .unlocked_ioctl = kvm_vm_ioctl,
  2146. #ifdef CONFIG_COMPAT
  2147. .compat_ioctl = kvm_vm_compat_ioctl,
  2148. #endif
  2149. .mmap = kvm_vm_mmap,
  2150. .llseek = noop_llseek,
  2151. };
  2152. static int kvm_dev_ioctl_create_vm(unsigned long type)
  2153. {
  2154. int r;
  2155. struct kvm *kvm;
  2156. kvm = kvm_create_vm(type);
  2157. if (IS_ERR(kvm))
  2158. return PTR_ERR(kvm);
  2159. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  2160. r = kvm_coalesced_mmio_init(kvm);
  2161. if (r < 0) {
  2162. kvm_put_kvm(kvm);
  2163. return r;
  2164. }
  2165. #endif
  2166. r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
  2167. if (r < 0)
  2168. kvm_put_kvm(kvm);
  2169. return r;
  2170. }
  2171. static long kvm_dev_ioctl_check_extension_generic(long arg)
  2172. {
  2173. switch (arg) {
  2174. case KVM_CAP_USER_MEMORY:
  2175. case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
  2176. case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
  2177. #ifdef CONFIG_KVM_APIC_ARCHITECTURE
  2178. case KVM_CAP_SET_BOOT_CPU_ID:
  2179. #endif
  2180. case KVM_CAP_INTERNAL_ERROR_DATA:
  2181. #ifdef CONFIG_HAVE_KVM_MSI
  2182. case KVM_CAP_SIGNAL_MSI:
  2183. #endif
  2184. #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
  2185. case KVM_CAP_IRQFD_RESAMPLE:
  2186. #endif
  2187. return 1;
  2188. #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
  2189. case KVM_CAP_IRQ_ROUTING:
  2190. return KVM_MAX_IRQ_ROUTES;
  2191. #endif
  2192. default:
  2193. break;
  2194. }
  2195. return kvm_dev_ioctl_check_extension(arg);
  2196. }
  2197. static long kvm_dev_ioctl(struct file *filp,
  2198. unsigned int ioctl, unsigned long arg)
  2199. {
  2200. long r = -EINVAL;
  2201. switch (ioctl) {
  2202. case KVM_GET_API_VERSION:
  2203. r = -EINVAL;
  2204. if (arg)
  2205. goto out;
  2206. r = KVM_API_VERSION;
  2207. break;
  2208. case KVM_CREATE_VM:
  2209. r = kvm_dev_ioctl_create_vm(arg);
  2210. break;
  2211. case KVM_CHECK_EXTENSION:
  2212. r = kvm_dev_ioctl_check_extension_generic(arg);
  2213. break;
  2214. case KVM_GET_VCPU_MMAP_SIZE:
  2215. r = -EINVAL;
  2216. if (arg)
  2217. goto out;
  2218. r = PAGE_SIZE; /* struct kvm_run */
  2219. #ifdef CONFIG_X86
  2220. r += PAGE_SIZE; /* pio data page */
  2221. #endif
  2222. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  2223. r += PAGE_SIZE; /* coalesced mmio ring page */
  2224. #endif
  2225. break;
  2226. case KVM_TRACE_ENABLE:
  2227. case KVM_TRACE_PAUSE:
  2228. case KVM_TRACE_DISABLE:
  2229. r = -EOPNOTSUPP;
  2230. break;
  2231. default:
  2232. return kvm_arch_dev_ioctl(filp, ioctl, arg);
  2233. }
  2234. out:
  2235. return r;
  2236. }
  2237. static struct file_operations kvm_chardev_ops = {
  2238. .unlocked_ioctl = kvm_dev_ioctl,
  2239. .compat_ioctl = kvm_dev_ioctl,
  2240. .llseek = noop_llseek,
  2241. };
  2242. static struct miscdevice kvm_dev = {
  2243. KVM_MINOR,
  2244. "kvm",
  2245. &kvm_chardev_ops,
  2246. };
  2247. static void hardware_enable_nolock(void *junk)
  2248. {
  2249. int cpu = raw_smp_processor_id();
  2250. int r;
  2251. if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
  2252. return;
  2253. cpumask_set_cpu(cpu, cpus_hardware_enabled);
  2254. r = kvm_arch_hardware_enable(NULL);
  2255. if (r) {
  2256. cpumask_clear_cpu(cpu, cpus_hardware_enabled);
  2257. atomic_inc(&hardware_enable_failed);
  2258. printk(KERN_INFO "kvm: enabling virtualization on "
  2259. "CPU%d failed\n", cpu);
  2260. }
  2261. }
  2262. static void hardware_enable(void *junk)
  2263. {
  2264. raw_spin_lock(&kvm_lock);
  2265. hardware_enable_nolock(junk);
  2266. raw_spin_unlock(&kvm_lock);
  2267. }
  2268. static void hardware_disable_nolock(void *junk)
  2269. {
  2270. int cpu = raw_smp_processor_id();
  2271. if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
  2272. return;
  2273. cpumask_clear_cpu(cpu, cpus_hardware_enabled);
  2274. kvm_arch_hardware_disable(NULL);
  2275. }
  2276. static void hardware_disable(void *junk)
  2277. {
  2278. raw_spin_lock(&kvm_lock);
  2279. hardware_disable_nolock(junk);
  2280. raw_spin_unlock(&kvm_lock);
  2281. }
  2282. static void hardware_disable_all_nolock(void)
  2283. {
  2284. BUG_ON(!kvm_usage_count);
  2285. kvm_usage_count--;
  2286. if (!kvm_usage_count)
  2287. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2288. }
  2289. static void hardware_disable_all(void)
  2290. {
  2291. raw_spin_lock(&kvm_lock);
  2292. hardware_disable_all_nolock();
  2293. raw_spin_unlock(&kvm_lock);
  2294. }
  2295. static int hardware_enable_all(void)
  2296. {
  2297. int r = 0;
  2298. raw_spin_lock(&kvm_lock);
  2299. kvm_usage_count++;
  2300. if (kvm_usage_count == 1) {
  2301. atomic_set(&hardware_enable_failed, 0);
  2302. on_each_cpu(hardware_enable_nolock, NULL, 1);
  2303. if (atomic_read(&hardware_enable_failed)) {
  2304. hardware_disable_all_nolock();
  2305. r = -EBUSY;
  2306. }
  2307. }
  2308. raw_spin_unlock(&kvm_lock);
  2309. return r;
  2310. }
  2311. static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  2312. void *v)
  2313. {
  2314. int cpu = (long)v;
  2315. if (!kvm_usage_count)
  2316. return NOTIFY_OK;
  2317. val &= ~CPU_TASKS_FROZEN;
  2318. switch (val) {
  2319. case CPU_DYING:
  2320. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  2321. cpu);
  2322. hardware_disable(NULL);
  2323. break;
  2324. case CPU_STARTING:
  2325. printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
  2326. cpu);
  2327. hardware_enable(NULL);
  2328. break;
  2329. }
  2330. return NOTIFY_OK;
  2331. }
  2332. static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
  2333. void *v)
  2334. {
  2335. /*
  2336. * Some (well, at least mine) BIOSes hang on reboot if
  2337. * in vmx root mode.
  2338. *
  2339. * And Intel TXT required VMX off for all cpu when system shutdown.
  2340. */
  2341. printk(KERN_INFO "kvm: exiting hardware virtualization\n");
  2342. kvm_rebooting = true;
  2343. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2344. return NOTIFY_OK;
  2345. }
  2346. static struct notifier_block kvm_reboot_notifier = {
  2347. .notifier_call = kvm_reboot,
  2348. .priority = 0,
  2349. };
  2350. static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
  2351. {
  2352. int i;
  2353. for (i = 0; i < bus->dev_count; i++) {
  2354. struct kvm_io_device *pos = bus->range[i].dev;
  2355. kvm_iodevice_destructor(pos);
  2356. }
  2357. kfree(bus);
  2358. }
  2359. static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
  2360. {
  2361. const struct kvm_io_range *r1 = p1;
  2362. const struct kvm_io_range *r2 = p2;
  2363. if (r1->addr < r2->addr)
  2364. return -1;
  2365. if (r1->addr + r1->len > r2->addr + r2->len)
  2366. return 1;
  2367. return 0;
  2368. }
  2369. static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
  2370. gpa_t addr, int len)
  2371. {
  2372. bus->range[bus->dev_count++] = (struct kvm_io_range) {
  2373. .addr = addr,
  2374. .len = len,
  2375. .dev = dev,
  2376. };
  2377. sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
  2378. kvm_io_bus_sort_cmp, NULL);
  2379. return 0;
  2380. }
  2381. static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
  2382. gpa_t addr, int len)
  2383. {
  2384. struct kvm_io_range *range, key;
  2385. int off;
  2386. key = (struct kvm_io_range) {
  2387. .addr = addr,
  2388. .len = len,
  2389. };
  2390. range = bsearch(&key, bus->range, bus->dev_count,
  2391. sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
  2392. if (range == NULL)
  2393. return -ENOENT;
  2394. off = range - bus->range;
  2395. while (off > 0 && kvm_io_bus_sort_cmp(&key, &bus->range[off-1]) == 0)
  2396. off--;
  2397. return off;
  2398. }
  2399. /* kvm_io_bus_write - called under kvm->slots_lock */
  2400. int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  2401. int len, const void *val)
  2402. {
  2403. int idx;
  2404. struct kvm_io_bus *bus;
  2405. struct kvm_io_range range;
  2406. range = (struct kvm_io_range) {
  2407. .addr = addr,
  2408. .len = len,
  2409. };
  2410. bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
  2411. idx = kvm_io_bus_get_first_dev(bus, addr, len);
  2412. if (idx < 0)
  2413. return -EOPNOTSUPP;
  2414. while (idx < bus->dev_count &&
  2415. kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
  2416. if (!kvm_iodevice_write(bus->range[idx].dev, addr, len, val))
  2417. return 0;
  2418. idx++;
  2419. }
  2420. return -EOPNOTSUPP;
  2421. }
  2422. /* kvm_io_bus_read - called under kvm->slots_lock */
  2423. int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  2424. int len, void *val)
  2425. {
  2426. int idx;
  2427. struct kvm_io_bus *bus;
  2428. struct kvm_io_range range;
  2429. range = (struct kvm_io_range) {
  2430. .addr = addr,
  2431. .len = len,
  2432. };
  2433. bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
  2434. idx = kvm_io_bus_get_first_dev(bus, addr, len);
  2435. if (idx < 0)
  2436. return -EOPNOTSUPP;
  2437. while (idx < bus->dev_count &&
  2438. kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
  2439. if (!kvm_iodevice_read(bus->range[idx].dev, addr, len, val))
  2440. return 0;
  2441. idx++;
  2442. }
  2443. return -EOPNOTSUPP;
  2444. }
  2445. /* Caller must hold slots_lock. */
  2446. int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  2447. int len, struct kvm_io_device *dev)
  2448. {
  2449. struct kvm_io_bus *new_bus, *bus;
  2450. bus = kvm->buses[bus_idx];
  2451. if (bus->dev_count > NR_IOBUS_DEVS - 1)
  2452. return -ENOSPC;
  2453. new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
  2454. sizeof(struct kvm_io_range)), GFP_KERNEL);
  2455. if (!new_bus)
  2456. return -ENOMEM;
  2457. memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
  2458. sizeof(struct kvm_io_range)));
  2459. kvm_io_bus_insert_dev(new_bus, dev, addr, len);
  2460. rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
  2461. synchronize_srcu_expedited(&kvm->srcu);
  2462. kfree(bus);
  2463. return 0;
  2464. }
  2465. /* Caller must hold slots_lock. */
  2466. int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
  2467. struct kvm_io_device *dev)
  2468. {
  2469. int i, r;
  2470. struct kvm_io_bus *new_bus, *bus;
  2471. bus = kvm->buses[bus_idx];
  2472. r = -ENOENT;
  2473. for (i = 0; i < bus->dev_count; i++)
  2474. if (bus->range[i].dev == dev) {
  2475. r = 0;
  2476. break;
  2477. }
  2478. if (r)
  2479. return r;
  2480. new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
  2481. sizeof(struct kvm_io_range)), GFP_KERNEL);
  2482. if (!new_bus)
  2483. return -ENOMEM;
  2484. memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
  2485. new_bus->dev_count--;
  2486. memcpy(new_bus->range + i, bus->range + i + 1,
  2487. (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
  2488. rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
  2489. synchronize_srcu_expedited(&kvm->srcu);
  2490. kfree(bus);
  2491. return r;
  2492. }
  2493. static struct notifier_block kvm_cpu_notifier = {
  2494. .notifier_call = kvm_cpu_hotplug,
  2495. };
  2496. static int vm_stat_get(void *_offset, u64 *val)
  2497. {
  2498. unsigned offset = (long)_offset;
  2499. struct kvm *kvm;
  2500. *val = 0;
  2501. raw_spin_lock(&kvm_lock);
  2502. list_for_each_entry(kvm, &vm_list, vm_list)
  2503. *val += *(u32 *)((void *)kvm + offset);
  2504. raw_spin_unlock(&kvm_lock);
  2505. return 0;
  2506. }
  2507. DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
  2508. static int vcpu_stat_get(void *_offset, u64 *val)
  2509. {
  2510. unsigned offset = (long)_offset;
  2511. struct kvm *kvm;
  2512. struct kvm_vcpu *vcpu;
  2513. int i;
  2514. *val = 0;
  2515. raw_spin_lock(&kvm_lock);
  2516. list_for_each_entry(kvm, &vm_list, vm_list)
  2517. kvm_for_each_vcpu(i, vcpu, kvm)
  2518. *val += *(u32 *)((void *)vcpu + offset);
  2519. raw_spin_unlock(&kvm_lock);
  2520. return 0;
  2521. }
  2522. DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
  2523. static const struct file_operations *stat_fops[] = {
  2524. [KVM_STAT_VCPU] = &vcpu_stat_fops,
  2525. [KVM_STAT_VM] = &vm_stat_fops,
  2526. };
  2527. static int kvm_init_debug(void)
  2528. {
  2529. int r = -EFAULT;
  2530. struct kvm_stats_debugfs_item *p;
  2531. kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
  2532. if (kvm_debugfs_dir == NULL)
  2533. goto out;
  2534. for (p = debugfs_entries; p->name; ++p) {
  2535. p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
  2536. (void *)(long)p->offset,
  2537. stat_fops[p->kind]);
  2538. if (p->dentry == NULL)
  2539. goto out_dir;
  2540. }
  2541. return 0;
  2542. out_dir:
  2543. debugfs_remove_recursive(kvm_debugfs_dir);
  2544. out:
  2545. return r;
  2546. }
  2547. static void kvm_exit_debug(void)
  2548. {
  2549. struct kvm_stats_debugfs_item *p;
  2550. for (p = debugfs_entries; p->name; ++p)
  2551. debugfs_remove(p->dentry);
  2552. debugfs_remove(kvm_debugfs_dir);
  2553. }
  2554. static int kvm_suspend(void)
  2555. {
  2556. if (kvm_usage_count)
  2557. hardware_disable_nolock(NULL);
  2558. return 0;
  2559. }
  2560. static void kvm_resume(void)
  2561. {
  2562. if (kvm_usage_count) {
  2563. WARN_ON(raw_spin_is_locked(&kvm_lock));
  2564. hardware_enable_nolock(NULL);
  2565. }
  2566. }
  2567. static struct syscore_ops kvm_syscore_ops = {
  2568. .suspend = kvm_suspend,
  2569. .resume = kvm_resume,
  2570. };
  2571. static inline
  2572. struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
  2573. {
  2574. return container_of(pn, struct kvm_vcpu, preempt_notifier);
  2575. }
  2576. static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
  2577. {
  2578. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  2579. if (vcpu->preempted)
  2580. vcpu->preempted = false;
  2581. kvm_arch_vcpu_load(vcpu, cpu);
  2582. }
  2583. static void kvm_sched_out(struct preempt_notifier *pn,
  2584. struct task_struct *next)
  2585. {
  2586. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  2587. if (current->state == TASK_RUNNING)
  2588. vcpu->preempted = true;
  2589. kvm_arch_vcpu_put(vcpu);
  2590. }
  2591. int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
  2592. struct module *module)
  2593. {
  2594. int r;
  2595. int cpu;
  2596. r = kvm_irqfd_init();
  2597. if (r)
  2598. goto out_irqfd;
  2599. r = kvm_arch_init(opaque);
  2600. if (r)
  2601. goto out_fail;
  2602. if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
  2603. r = -ENOMEM;
  2604. goto out_free_0;
  2605. }
  2606. r = kvm_arch_hardware_setup();
  2607. if (r < 0)
  2608. goto out_free_0a;
  2609. for_each_online_cpu(cpu) {
  2610. smp_call_function_single(cpu,
  2611. kvm_arch_check_processor_compat,
  2612. &r, 1);
  2613. if (r < 0)
  2614. goto out_free_1;
  2615. }
  2616. r = register_cpu_notifier(&kvm_cpu_notifier);
  2617. if (r)
  2618. goto out_free_2;
  2619. register_reboot_notifier(&kvm_reboot_notifier);
  2620. /* A kmem cache lets us meet the alignment requirements of fx_save. */
  2621. if (!vcpu_align)
  2622. vcpu_align = __alignof__(struct kvm_vcpu);
  2623. kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
  2624. 0, NULL);
  2625. if (!kvm_vcpu_cache) {
  2626. r = -ENOMEM;
  2627. goto out_free_3;
  2628. }
  2629. r = kvm_async_pf_init();
  2630. if (r)
  2631. goto out_free;
  2632. kvm_chardev_ops.owner = module;
  2633. kvm_vm_fops.owner = module;
  2634. kvm_vcpu_fops.owner = module;
  2635. r = misc_register(&kvm_dev);
  2636. if (r) {
  2637. printk(KERN_ERR "kvm: misc device register failed\n");
  2638. goto out_unreg;
  2639. }
  2640. register_syscore_ops(&kvm_syscore_ops);
  2641. kvm_preempt_ops.sched_in = kvm_sched_in;
  2642. kvm_preempt_ops.sched_out = kvm_sched_out;
  2643. r = kvm_init_debug();
  2644. if (r) {
  2645. printk(KERN_ERR "kvm: create debugfs files failed\n");
  2646. goto out_undebugfs;
  2647. }
  2648. return 0;
  2649. out_undebugfs:
  2650. unregister_syscore_ops(&kvm_syscore_ops);
  2651. out_unreg:
  2652. kvm_async_pf_deinit();
  2653. out_free:
  2654. kmem_cache_destroy(kvm_vcpu_cache);
  2655. out_free_3:
  2656. unregister_reboot_notifier(&kvm_reboot_notifier);
  2657. unregister_cpu_notifier(&kvm_cpu_notifier);
  2658. out_free_2:
  2659. out_free_1:
  2660. kvm_arch_hardware_unsetup();
  2661. out_free_0a:
  2662. free_cpumask_var(cpus_hardware_enabled);
  2663. out_free_0:
  2664. kvm_arch_exit();
  2665. out_fail:
  2666. kvm_irqfd_exit();
  2667. out_irqfd:
  2668. return r;
  2669. }
  2670. EXPORT_SYMBOL_GPL(kvm_init);
  2671. void kvm_exit(void)
  2672. {
  2673. kvm_exit_debug();
  2674. misc_deregister(&kvm_dev);
  2675. kmem_cache_destroy(kvm_vcpu_cache);
  2676. kvm_async_pf_deinit();
  2677. unregister_syscore_ops(&kvm_syscore_ops);
  2678. unregister_reboot_notifier(&kvm_reboot_notifier);
  2679. unregister_cpu_notifier(&kvm_cpu_notifier);
  2680. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2681. kvm_arch_hardware_unsetup();
  2682. kvm_arch_exit();
  2683. kvm_irqfd_exit();
  2684. free_cpumask_var(cpus_hardware_enabled);
  2685. }
  2686. EXPORT_SYMBOL_GPL(kvm_exit);