kvm_main.c 67 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  9. *
  10. * Authors:
  11. * Avi Kivity <avi@qumranet.com>
  12. * Yaniv Kamay <yaniv@qumranet.com>
  13. *
  14. * This work is licensed under the terms of the GNU GPL, version 2. See
  15. * the COPYING file in the top-level directory.
  16. *
  17. */
  18. #include "iodev.h"
  19. #include <linux/kvm_host.h>
  20. #include <linux/kvm.h>
  21. #include <linux/module.h>
  22. #include <linux/errno.h>
  23. #include <linux/percpu.h>
  24. #include <linux/mm.h>
  25. #include <linux/miscdevice.h>
  26. #include <linux/vmalloc.h>
  27. #include <linux/reboot.h>
  28. #include <linux/debugfs.h>
  29. #include <linux/highmem.h>
  30. #include <linux/file.h>
  31. #include <linux/syscore_ops.h>
  32. #include <linux/cpu.h>
  33. #include <linux/sched.h>
  34. #include <linux/cpumask.h>
  35. #include <linux/smp.h>
  36. #include <linux/anon_inodes.h>
  37. #include <linux/profile.h>
  38. #include <linux/kvm_para.h>
  39. #include <linux/pagemap.h>
  40. #include <linux/mman.h>
  41. #include <linux/swap.h>
  42. #include <linux/bitops.h>
  43. #include <linux/spinlock.h>
  44. #include <linux/compat.h>
  45. #include <linux/srcu.h>
  46. #include <linux/hugetlb.h>
  47. #include <linux/slab.h>
  48. #include <linux/sort.h>
  49. #include <linux/bsearch.h>
  50. #include <asm/processor.h>
  51. #include <asm/io.h>
  52. #include <asm/uaccess.h>
  53. #include <asm/pgtable.h>
  54. #include "coalesced_mmio.h"
  55. #include "async_pf.h"
  56. #define CREATE_TRACE_POINTS
  57. #include <trace/events/kvm.h>
  58. MODULE_AUTHOR("Qumranet");
  59. MODULE_LICENSE("GPL");
  60. /*
  61. * Ordering of locks:
  62. *
  63. * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
  64. */
  65. DEFINE_RAW_SPINLOCK(kvm_lock);
  66. LIST_HEAD(vm_list);
  67. static cpumask_var_t cpus_hardware_enabled;
  68. static int kvm_usage_count = 0;
  69. static atomic_t hardware_enable_failed;
  70. struct kmem_cache *kvm_vcpu_cache;
  71. EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  72. static __read_mostly struct preempt_ops kvm_preempt_ops;
  73. struct dentry *kvm_debugfs_dir;
  74. static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  75. unsigned long arg);
  76. #ifdef CONFIG_COMPAT
  77. static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
  78. unsigned long arg);
  79. #endif
  80. static int hardware_enable_all(void);
  81. static void hardware_disable_all(void);
  82. static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
  83. bool kvm_rebooting;
  84. EXPORT_SYMBOL_GPL(kvm_rebooting);
  85. static bool largepages_enabled = true;
  86. bool kvm_is_mmio_pfn(pfn_t pfn)
  87. {
  88. if (pfn_valid(pfn)) {
  89. int reserved;
  90. struct page *tail = pfn_to_page(pfn);
  91. struct page *head = compound_trans_head(tail);
  92. reserved = PageReserved(head);
  93. if (head != tail) {
  94. /*
  95. * "head" is not a dangling pointer
  96. * (compound_trans_head takes care of that)
  97. * but the hugepage may have been splitted
  98. * from under us (and we may not hold a
  99. * reference count on the head page so it can
  100. * be reused before we run PageReferenced), so
  101. * we've to check PageTail before returning
  102. * what we just read.
  103. */
  104. smp_rmb();
  105. if (PageTail(tail))
  106. return reserved;
  107. }
  108. return PageReserved(tail);
  109. }
  110. return true;
  111. }
  112. /*
  113. * Switches to specified vcpu, until a matching vcpu_put()
  114. */
  115. int vcpu_load(struct kvm_vcpu *vcpu)
  116. {
  117. int cpu;
  118. if (mutex_lock_killable(&vcpu->mutex))
  119. return -EINTR;
  120. if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
  121. /* The thread running this VCPU changed. */
  122. struct pid *oldpid = vcpu->pid;
  123. struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
  124. rcu_assign_pointer(vcpu->pid, newpid);
  125. synchronize_rcu();
  126. put_pid(oldpid);
  127. }
  128. cpu = get_cpu();
  129. preempt_notifier_register(&vcpu->preempt_notifier);
  130. kvm_arch_vcpu_load(vcpu, cpu);
  131. put_cpu();
  132. return 0;
  133. }
  134. void vcpu_put(struct kvm_vcpu *vcpu)
  135. {
  136. preempt_disable();
  137. kvm_arch_vcpu_put(vcpu);
  138. preempt_notifier_unregister(&vcpu->preempt_notifier);
  139. preempt_enable();
  140. mutex_unlock(&vcpu->mutex);
  141. }
  142. static void ack_flush(void *_completed)
  143. {
  144. }
  145. static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
  146. {
  147. int i, cpu, me;
  148. cpumask_var_t cpus;
  149. bool called = true;
  150. struct kvm_vcpu *vcpu;
  151. zalloc_cpumask_var(&cpus, GFP_ATOMIC);
  152. me = get_cpu();
  153. kvm_for_each_vcpu(i, vcpu, kvm) {
  154. kvm_make_request(req, vcpu);
  155. cpu = vcpu->cpu;
  156. /* Set ->requests bit before we read ->mode */
  157. smp_mb();
  158. if (cpus != NULL && cpu != -1 && cpu != me &&
  159. kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
  160. cpumask_set_cpu(cpu, cpus);
  161. }
  162. if (unlikely(cpus == NULL))
  163. smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
  164. else if (!cpumask_empty(cpus))
  165. smp_call_function_many(cpus, ack_flush, NULL, 1);
  166. else
  167. called = false;
  168. put_cpu();
  169. free_cpumask_var(cpus);
  170. return called;
  171. }
  172. void kvm_flush_remote_tlbs(struct kvm *kvm)
  173. {
  174. long dirty_count = kvm->tlbs_dirty;
  175. smp_mb();
  176. if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
  177. ++kvm->stat.remote_tlb_flush;
  178. cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
  179. }
  180. void kvm_reload_remote_mmus(struct kvm *kvm)
  181. {
  182. make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
  183. }
  184. void kvm_make_mclock_inprogress_request(struct kvm *kvm)
  185. {
  186. make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
  187. }
  188. void kvm_make_update_eoibitmap_request(struct kvm *kvm)
  189. {
  190. make_all_cpus_request(kvm, KVM_REQ_EOIBITMAP);
  191. }
  192. int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
  193. {
  194. struct page *page;
  195. int r;
  196. mutex_init(&vcpu->mutex);
  197. vcpu->cpu = -1;
  198. vcpu->kvm = kvm;
  199. vcpu->vcpu_id = id;
  200. vcpu->pid = NULL;
  201. init_waitqueue_head(&vcpu->wq);
  202. kvm_async_pf_vcpu_init(vcpu);
  203. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  204. if (!page) {
  205. r = -ENOMEM;
  206. goto fail;
  207. }
  208. vcpu->run = page_address(page);
  209. kvm_vcpu_set_in_spin_loop(vcpu, false);
  210. kvm_vcpu_set_dy_eligible(vcpu, false);
  211. r = kvm_arch_vcpu_init(vcpu);
  212. if (r < 0)
  213. goto fail_free_run;
  214. return 0;
  215. fail_free_run:
  216. free_page((unsigned long)vcpu->run);
  217. fail:
  218. return r;
  219. }
  220. EXPORT_SYMBOL_GPL(kvm_vcpu_init);
  221. void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
  222. {
  223. put_pid(vcpu->pid);
  224. kvm_arch_vcpu_uninit(vcpu);
  225. free_page((unsigned long)vcpu->run);
  226. }
  227. EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
  228. #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
  229. static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
  230. {
  231. return container_of(mn, struct kvm, mmu_notifier);
  232. }
  233. static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
  234. struct mm_struct *mm,
  235. unsigned long address)
  236. {
  237. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  238. int need_tlb_flush, idx;
  239. /*
  240. * When ->invalidate_page runs, the linux pte has been zapped
  241. * already but the page is still allocated until
  242. * ->invalidate_page returns. So if we increase the sequence
  243. * here the kvm page fault will notice if the spte can't be
  244. * established because the page is going to be freed. If
  245. * instead the kvm page fault establishes the spte before
  246. * ->invalidate_page runs, kvm_unmap_hva will release it
  247. * before returning.
  248. *
  249. * The sequence increase only need to be seen at spin_unlock
  250. * time, and not at spin_lock time.
  251. *
  252. * Increasing the sequence after the spin_unlock would be
  253. * unsafe because the kvm page fault could then establish the
  254. * pte after kvm_unmap_hva returned, without noticing the page
  255. * is going to be freed.
  256. */
  257. idx = srcu_read_lock(&kvm->srcu);
  258. spin_lock(&kvm->mmu_lock);
  259. kvm->mmu_notifier_seq++;
  260. need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
  261. /* we've to flush the tlb before the pages can be freed */
  262. if (need_tlb_flush)
  263. kvm_flush_remote_tlbs(kvm);
  264. spin_unlock(&kvm->mmu_lock);
  265. srcu_read_unlock(&kvm->srcu, idx);
  266. }
  267. static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
  268. struct mm_struct *mm,
  269. unsigned long address,
  270. pte_t pte)
  271. {
  272. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  273. int idx;
  274. idx = srcu_read_lock(&kvm->srcu);
  275. spin_lock(&kvm->mmu_lock);
  276. kvm->mmu_notifier_seq++;
  277. kvm_set_spte_hva(kvm, address, pte);
  278. spin_unlock(&kvm->mmu_lock);
  279. srcu_read_unlock(&kvm->srcu, idx);
  280. }
  281. static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
  282. struct mm_struct *mm,
  283. unsigned long start,
  284. unsigned long end)
  285. {
  286. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  287. int need_tlb_flush = 0, idx;
  288. idx = srcu_read_lock(&kvm->srcu);
  289. spin_lock(&kvm->mmu_lock);
  290. /*
  291. * The count increase must become visible at unlock time as no
  292. * spte can be established without taking the mmu_lock and
  293. * count is also read inside the mmu_lock critical section.
  294. */
  295. kvm->mmu_notifier_count++;
  296. need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
  297. need_tlb_flush |= kvm->tlbs_dirty;
  298. /* we've to flush the tlb before the pages can be freed */
  299. if (need_tlb_flush)
  300. kvm_flush_remote_tlbs(kvm);
  301. spin_unlock(&kvm->mmu_lock);
  302. srcu_read_unlock(&kvm->srcu, idx);
  303. }
  304. static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
  305. struct mm_struct *mm,
  306. unsigned long start,
  307. unsigned long end)
  308. {
  309. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  310. spin_lock(&kvm->mmu_lock);
  311. /*
  312. * This sequence increase will notify the kvm page fault that
  313. * the page that is going to be mapped in the spte could have
  314. * been freed.
  315. */
  316. kvm->mmu_notifier_seq++;
  317. smp_wmb();
  318. /*
  319. * The above sequence increase must be visible before the
  320. * below count decrease, which is ensured by the smp_wmb above
  321. * in conjunction with the smp_rmb in mmu_notifier_retry().
  322. */
  323. kvm->mmu_notifier_count--;
  324. spin_unlock(&kvm->mmu_lock);
  325. BUG_ON(kvm->mmu_notifier_count < 0);
  326. }
  327. static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
  328. struct mm_struct *mm,
  329. unsigned long address)
  330. {
  331. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  332. int young, idx;
  333. idx = srcu_read_lock(&kvm->srcu);
  334. spin_lock(&kvm->mmu_lock);
  335. young = kvm_age_hva(kvm, address);
  336. if (young)
  337. kvm_flush_remote_tlbs(kvm);
  338. spin_unlock(&kvm->mmu_lock);
  339. srcu_read_unlock(&kvm->srcu, idx);
  340. return young;
  341. }
  342. static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
  343. struct mm_struct *mm,
  344. unsigned long address)
  345. {
  346. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  347. int young, idx;
  348. idx = srcu_read_lock(&kvm->srcu);
  349. spin_lock(&kvm->mmu_lock);
  350. young = kvm_test_age_hva(kvm, address);
  351. spin_unlock(&kvm->mmu_lock);
  352. srcu_read_unlock(&kvm->srcu, idx);
  353. return young;
  354. }
  355. static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
  356. struct mm_struct *mm)
  357. {
  358. struct kvm *kvm = mmu_notifier_to_kvm(mn);
  359. int idx;
  360. idx = srcu_read_lock(&kvm->srcu);
  361. kvm_arch_flush_shadow_all(kvm);
  362. srcu_read_unlock(&kvm->srcu, idx);
  363. }
  364. static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
  365. .invalidate_page = kvm_mmu_notifier_invalidate_page,
  366. .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
  367. .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
  368. .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
  369. .test_young = kvm_mmu_notifier_test_young,
  370. .change_pte = kvm_mmu_notifier_change_pte,
  371. .release = kvm_mmu_notifier_release,
  372. };
  373. static int kvm_init_mmu_notifier(struct kvm *kvm)
  374. {
  375. kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
  376. return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
  377. }
  378. #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
  379. static int kvm_init_mmu_notifier(struct kvm *kvm)
  380. {
  381. return 0;
  382. }
  383. #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
  384. static void kvm_init_memslots_id(struct kvm *kvm)
  385. {
  386. int i;
  387. struct kvm_memslots *slots = kvm->memslots;
  388. for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
  389. slots->id_to_index[i] = slots->memslots[i].id = i;
  390. }
  391. static struct kvm *kvm_create_vm(unsigned long type)
  392. {
  393. int r, i;
  394. struct kvm *kvm = kvm_arch_alloc_vm();
  395. if (!kvm)
  396. return ERR_PTR(-ENOMEM);
  397. r = kvm_arch_init_vm(kvm, type);
  398. if (r)
  399. goto out_err_nodisable;
  400. r = hardware_enable_all();
  401. if (r)
  402. goto out_err_nodisable;
  403. #ifdef CONFIG_HAVE_KVM_IRQCHIP
  404. INIT_HLIST_HEAD(&kvm->mask_notifier_list);
  405. INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
  406. #endif
  407. BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
  408. r = -ENOMEM;
  409. kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
  410. if (!kvm->memslots)
  411. goto out_err_nosrcu;
  412. kvm_init_memslots_id(kvm);
  413. if (init_srcu_struct(&kvm->srcu))
  414. goto out_err_nosrcu;
  415. for (i = 0; i < KVM_NR_BUSES; i++) {
  416. kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
  417. GFP_KERNEL);
  418. if (!kvm->buses[i])
  419. goto out_err;
  420. }
  421. spin_lock_init(&kvm->mmu_lock);
  422. kvm->mm = current->mm;
  423. atomic_inc(&kvm->mm->mm_count);
  424. kvm_eventfd_init(kvm);
  425. mutex_init(&kvm->lock);
  426. mutex_init(&kvm->irq_lock);
  427. mutex_init(&kvm->slots_lock);
  428. atomic_set(&kvm->users_count, 1);
  429. r = kvm_init_mmu_notifier(kvm);
  430. if (r)
  431. goto out_err;
  432. raw_spin_lock(&kvm_lock);
  433. list_add(&kvm->vm_list, &vm_list);
  434. raw_spin_unlock(&kvm_lock);
  435. return kvm;
  436. out_err:
  437. cleanup_srcu_struct(&kvm->srcu);
  438. out_err_nosrcu:
  439. hardware_disable_all();
  440. out_err_nodisable:
  441. for (i = 0; i < KVM_NR_BUSES; i++)
  442. kfree(kvm->buses[i]);
  443. kfree(kvm->memslots);
  444. kvm_arch_free_vm(kvm);
  445. return ERR_PTR(r);
  446. }
  447. /*
  448. * Avoid using vmalloc for a small buffer.
  449. * Should not be used when the size is statically known.
  450. */
  451. void *kvm_kvzalloc(unsigned long size)
  452. {
  453. if (size > PAGE_SIZE)
  454. return vzalloc(size);
  455. else
  456. return kzalloc(size, GFP_KERNEL);
  457. }
  458. void kvm_kvfree(const void *addr)
  459. {
  460. if (is_vmalloc_addr(addr))
  461. vfree(addr);
  462. else
  463. kfree(addr);
  464. }
  465. static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
  466. {
  467. if (!memslot->dirty_bitmap)
  468. return;
  469. kvm_kvfree(memslot->dirty_bitmap);
  470. memslot->dirty_bitmap = NULL;
  471. }
  472. /*
  473. * Free any memory in @free but not in @dont.
  474. */
  475. static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
  476. struct kvm_memory_slot *dont)
  477. {
  478. if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
  479. kvm_destroy_dirty_bitmap(free);
  480. kvm_arch_free_memslot(free, dont);
  481. free->npages = 0;
  482. }
  483. void kvm_free_physmem(struct kvm *kvm)
  484. {
  485. struct kvm_memslots *slots = kvm->memslots;
  486. struct kvm_memory_slot *memslot;
  487. kvm_for_each_memslot(memslot, slots)
  488. kvm_free_physmem_slot(memslot, NULL);
  489. kfree(kvm->memslots);
  490. }
  491. static void kvm_destroy_vm(struct kvm *kvm)
  492. {
  493. int i;
  494. struct mm_struct *mm = kvm->mm;
  495. kvm_arch_sync_events(kvm);
  496. raw_spin_lock(&kvm_lock);
  497. list_del(&kvm->vm_list);
  498. raw_spin_unlock(&kvm_lock);
  499. kvm_free_irq_routing(kvm);
  500. for (i = 0; i < KVM_NR_BUSES; i++)
  501. kvm_io_bus_destroy(kvm->buses[i]);
  502. kvm_coalesced_mmio_free(kvm);
  503. #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
  504. mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
  505. #else
  506. kvm_arch_flush_shadow_all(kvm);
  507. #endif
  508. kvm_arch_destroy_vm(kvm);
  509. kvm_free_physmem(kvm);
  510. cleanup_srcu_struct(&kvm->srcu);
  511. kvm_arch_free_vm(kvm);
  512. hardware_disable_all();
  513. mmdrop(mm);
  514. }
  515. void kvm_get_kvm(struct kvm *kvm)
  516. {
  517. atomic_inc(&kvm->users_count);
  518. }
  519. EXPORT_SYMBOL_GPL(kvm_get_kvm);
  520. void kvm_put_kvm(struct kvm *kvm)
  521. {
  522. if (atomic_dec_and_test(&kvm->users_count))
  523. kvm_destroy_vm(kvm);
  524. }
  525. EXPORT_SYMBOL_GPL(kvm_put_kvm);
  526. static int kvm_vm_release(struct inode *inode, struct file *filp)
  527. {
  528. struct kvm *kvm = filp->private_data;
  529. kvm_irqfd_release(kvm);
  530. kvm_put_kvm(kvm);
  531. return 0;
  532. }
  533. /*
  534. * Allocation size is twice as large as the actual dirty bitmap size.
  535. * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
  536. */
  537. static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
  538. {
  539. #ifndef CONFIG_S390
  540. unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
  541. memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
  542. if (!memslot->dirty_bitmap)
  543. return -ENOMEM;
  544. #endif /* !CONFIG_S390 */
  545. return 0;
  546. }
  547. static int cmp_memslot(const void *slot1, const void *slot2)
  548. {
  549. struct kvm_memory_slot *s1, *s2;
  550. s1 = (struct kvm_memory_slot *)slot1;
  551. s2 = (struct kvm_memory_slot *)slot2;
  552. if (s1->npages < s2->npages)
  553. return 1;
  554. if (s1->npages > s2->npages)
  555. return -1;
  556. return 0;
  557. }
  558. /*
  559. * Sort the memslots base on its size, so the larger slots
  560. * will get better fit.
  561. */
  562. static void sort_memslots(struct kvm_memslots *slots)
  563. {
  564. int i;
  565. sort(slots->memslots, KVM_MEM_SLOTS_NUM,
  566. sizeof(struct kvm_memory_slot), cmp_memslot, NULL);
  567. for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
  568. slots->id_to_index[slots->memslots[i].id] = i;
  569. }
  570. void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new,
  571. u64 last_generation)
  572. {
  573. if (new) {
  574. int id = new->id;
  575. struct kvm_memory_slot *old = id_to_memslot(slots, id);
  576. unsigned long npages = old->npages;
  577. *old = *new;
  578. if (new->npages != npages)
  579. sort_memslots(slots);
  580. }
  581. slots->generation = last_generation + 1;
  582. }
  583. static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
  584. {
  585. u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
  586. #ifdef KVM_CAP_READONLY_MEM
  587. valid_flags |= KVM_MEM_READONLY;
  588. #endif
  589. if (mem->flags & ~valid_flags)
  590. return -EINVAL;
  591. return 0;
  592. }
  593. static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
  594. struct kvm_memslots *slots, struct kvm_memory_slot *new)
  595. {
  596. struct kvm_memslots *old_memslots = kvm->memslots;
  597. update_memslots(slots, new, kvm->memslots->generation);
  598. rcu_assign_pointer(kvm->memslots, slots);
  599. synchronize_srcu_expedited(&kvm->srcu);
  600. return old_memslots;
  601. }
  602. /*
  603. * Allocate some memory and give it an address in the guest physical address
  604. * space.
  605. *
  606. * Discontiguous memory is allowed, mostly for framebuffers.
  607. *
  608. * Must be called holding mmap_sem for write.
  609. */
  610. int __kvm_set_memory_region(struct kvm *kvm,
  611. struct kvm_userspace_memory_region *mem,
  612. bool user_alloc)
  613. {
  614. int r;
  615. gfn_t base_gfn;
  616. unsigned long npages;
  617. struct kvm_memory_slot *slot;
  618. struct kvm_memory_slot old, new;
  619. struct kvm_memslots *slots = NULL, *old_memslots;
  620. bool old_iommu_mapped;
  621. r = check_memory_region_flags(mem);
  622. if (r)
  623. goto out;
  624. r = -EINVAL;
  625. /* General sanity checks */
  626. if (mem->memory_size & (PAGE_SIZE - 1))
  627. goto out;
  628. if (mem->guest_phys_addr & (PAGE_SIZE - 1))
  629. goto out;
  630. /* We can read the guest memory with __xxx_user() later on. */
  631. if (user_alloc &&
  632. ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
  633. !access_ok(VERIFY_WRITE,
  634. (void __user *)(unsigned long)mem->userspace_addr,
  635. mem->memory_size)))
  636. goto out;
  637. if (mem->slot >= KVM_MEM_SLOTS_NUM)
  638. goto out;
  639. if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
  640. goto out;
  641. slot = id_to_memslot(kvm->memslots, mem->slot);
  642. base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
  643. npages = mem->memory_size >> PAGE_SHIFT;
  644. r = -EINVAL;
  645. if (npages > KVM_MEM_MAX_NR_PAGES)
  646. goto out;
  647. if (!npages)
  648. mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
  649. new = old = *slot;
  650. new.id = mem->slot;
  651. new.base_gfn = base_gfn;
  652. new.npages = npages;
  653. new.flags = mem->flags;
  654. old_iommu_mapped = old.npages;
  655. /*
  656. * Disallow changing a memory slot's size or changing anything about
  657. * zero sized slots that doesn't involve making them non-zero.
  658. */
  659. r = -EINVAL;
  660. if (npages && old.npages && npages != old.npages)
  661. goto out;
  662. if (!npages && !old.npages)
  663. goto out;
  664. if ((npages && !old.npages) || (base_gfn != old.base_gfn)) {
  665. /* Check for overlaps */
  666. r = -EEXIST;
  667. kvm_for_each_memslot(slot, kvm->memslots) {
  668. if ((slot->id >= KVM_USER_MEM_SLOTS) ||
  669. (slot->id == mem->slot))
  670. continue;
  671. if (!((base_gfn + npages <= slot->base_gfn) ||
  672. (base_gfn >= slot->base_gfn + slot->npages)))
  673. goto out;
  674. }
  675. }
  676. /* Free page dirty bitmap if unneeded */
  677. if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
  678. new.dirty_bitmap = NULL;
  679. r = -ENOMEM;
  680. /*
  681. * Allocate if a slot is being created. If modifying a slot,
  682. * the userspace_addr cannot change.
  683. */
  684. if (!old.npages) {
  685. new.user_alloc = user_alloc;
  686. new.userspace_addr = mem->userspace_addr;
  687. if (kvm_arch_create_memslot(&new, npages))
  688. goto out_free;
  689. } else if (npages && mem->userspace_addr != old.userspace_addr) {
  690. r = -EINVAL;
  691. goto out_free;
  692. }
  693. /* Allocate page dirty bitmap if needed */
  694. if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
  695. if (kvm_create_dirty_bitmap(&new) < 0)
  696. goto out_free;
  697. }
  698. if (!npages || base_gfn != old.base_gfn) {
  699. r = -ENOMEM;
  700. slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
  701. GFP_KERNEL);
  702. if (!slots)
  703. goto out_free;
  704. slot = id_to_memslot(slots, mem->slot);
  705. slot->flags |= KVM_MEMSLOT_INVALID;
  706. old_memslots = install_new_memslots(kvm, slots, NULL);
  707. /* slot was deleted or moved, clear iommu mapping */
  708. kvm_iommu_unmap_pages(kvm, &old);
  709. old_iommu_mapped = false;
  710. /* From this point no new shadow pages pointing to a deleted,
  711. * or moved, memslot will be created.
  712. *
  713. * validation of sp->gfn happens in:
  714. * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
  715. * - kvm_is_visible_gfn (mmu_check_roots)
  716. */
  717. kvm_arch_flush_shadow_memslot(kvm, slot);
  718. slots = old_memslots;
  719. }
  720. r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
  721. if (r)
  722. goto out_slots;
  723. r = -ENOMEM;
  724. /*
  725. * We can re-use the old_memslots from above, the only difference
  726. * from the currently installed memslots is the invalid flag. This
  727. * will get overwritten by update_memslots anyway.
  728. */
  729. if (!slots) {
  730. slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
  731. GFP_KERNEL);
  732. if (!slots)
  733. goto out_free;
  734. }
  735. /*
  736. * IOMMU mapping: New slots need to be mapped. Old slots need to be
  737. * un-mapped and re-mapped if their base changes or if flags that the
  738. * iommu cares about change (read-only). Base change unmapping is
  739. * handled above with slot deletion, so we only unmap incompatible
  740. * flags here. Anything else the iommu might care about for existing
  741. * slots (size changes, userspace addr changes) is disallowed above,
  742. * so any other attribute changes getting here can be skipped.
  743. */
  744. if (npages) {
  745. if (old_iommu_mapped &&
  746. ((new.flags ^ old.flags) & KVM_MEM_READONLY)) {
  747. kvm_iommu_unmap_pages(kvm, &old);
  748. old_iommu_mapped = false;
  749. }
  750. if (!old_iommu_mapped) {
  751. r = kvm_iommu_map_pages(kvm, &new);
  752. if (r)
  753. goto out_slots;
  754. }
  755. }
  756. /* actual memory is freed via old in kvm_free_physmem_slot below */
  757. if (!npages) {
  758. new.dirty_bitmap = NULL;
  759. memset(&new.arch, 0, sizeof(new.arch));
  760. }
  761. old_memslots = install_new_memslots(kvm, slots, &new);
  762. kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
  763. kvm_free_physmem_slot(&old, &new);
  764. kfree(old_memslots);
  765. return 0;
  766. out_slots:
  767. kfree(slots);
  768. out_free:
  769. kvm_free_physmem_slot(&new, &old);
  770. out:
  771. return r;
  772. }
  773. EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
  774. int kvm_set_memory_region(struct kvm *kvm,
  775. struct kvm_userspace_memory_region *mem,
  776. bool user_alloc)
  777. {
  778. int r;
  779. mutex_lock(&kvm->slots_lock);
  780. r = __kvm_set_memory_region(kvm, mem, user_alloc);
  781. mutex_unlock(&kvm->slots_lock);
  782. return r;
  783. }
  784. EXPORT_SYMBOL_GPL(kvm_set_memory_region);
  785. int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  786. struct
  787. kvm_userspace_memory_region *mem,
  788. bool user_alloc)
  789. {
  790. if (mem->slot >= KVM_USER_MEM_SLOTS)
  791. return -EINVAL;
  792. return kvm_set_memory_region(kvm, mem, user_alloc);
  793. }
  794. int kvm_get_dirty_log(struct kvm *kvm,
  795. struct kvm_dirty_log *log, int *is_dirty)
  796. {
  797. struct kvm_memory_slot *memslot;
  798. int r, i;
  799. unsigned long n;
  800. unsigned long any = 0;
  801. r = -EINVAL;
  802. if (log->slot >= KVM_USER_MEM_SLOTS)
  803. goto out;
  804. memslot = id_to_memslot(kvm->memslots, log->slot);
  805. r = -ENOENT;
  806. if (!memslot->dirty_bitmap)
  807. goto out;
  808. n = kvm_dirty_bitmap_bytes(memslot);
  809. for (i = 0; !any && i < n/sizeof(long); ++i)
  810. any = memslot->dirty_bitmap[i];
  811. r = -EFAULT;
  812. if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
  813. goto out;
  814. if (any)
  815. *is_dirty = 1;
  816. r = 0;
  817. out:
  818. return r;
  819. }
  820. bool kvm_largepages_enabled(void)
  821. {
  822. return largepages_enabled;
  823. }
  824. void kvm_disable_largepages(void)
  825. {
  826. largepages_enabled = false;
  827. }
  828. EXPORT_SYMBOL_GPL(kvm_disable_largepages);
  829. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  830. {
  831. return __gfn_to_memslot(kvm_memslots(kvm), gfn);
  832. }
  833. EXPORT_SYMBOL_GPL(gfn_to_memslot);
  834. int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
  835. {
  836. struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
  837. if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
  838. memslot->flags & KVM_MEMSLOT_INVALID)
  839. return 0;
  840. return 1;
  841. }
  842. EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
  843. unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
  844. {
  845. struct vm_area_struct *vma;
  846. unsigned long addr, size;
  847. size = PAGE_SIZE;
  848. addr = gfn_to_hva(kvm, gfn);
  849. if (kvm_is_error_hva(addr))
  850. return PAGE_SIZE;
  851. down_read(&current->mm->mmap_sem);
  852. vma = find_vma(current->mm, addr);
  853. if (!vma)
  854. goto out;
  855. size = vma_kernel_pagesize(vma);
  856. out:
  857. up_read(&current->mm->mmap_sem);
  858. return size;
  859. }
  860. static bool memslot_is_readonly(struct kvm_memory_slot *slot)
  861. {
  862. return slot->flags & KVM_MEM_READONLY;
  863. }
  864. static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
  865. gfn_t *nr_pages, bool write)
  866. {
  867. if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
  868. return KVM_HVA_ERR_BAD;
  869. if (memslot_is_readonly(slot) && write)
  870. return KVM_HVA_ERR_RO_BAD;
  871. if (nr_pages)
  872. *nr_pages = slot->npages - (gfn - slot->base_gfn);
  873. return __gfn_to_hva_memslot(slot, gfn);
  874. }
  875. static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
  876. gfn_t *nr_pages)
  877. {
  878. return __gfn_to_hva_many(slot, gfn, nr_pages, true);
  879. }
  880. unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
  881. gfn_t gfn)
  882. {
  883. return gfn_to_hva_many(slot, gfn, NULL);
  884. }
  885. EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
  886. unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
  887. {
  888. return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
  889. }
  890. EXPORT_SYMBOL_GPL(gfn_to_hva);
  891. /*
  892. * The hva returned by this function is only allowed to be read.
  893. * It should pair with kvm_read_hva() or kvm_read_hva_atomic().
  894. */
  895. static unsigned long gfn_to_hva_read(struct kvm *kvm, gfn_t gfn)
  896. {
  897. return __gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL, false);
  898. }
  899. static int kvm_read_hva(void *data, void __user *hva, int len)
  900. {
  901. return __copy_from_user(data, hva, len);
  902. }
  903. static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
  904. {
  905. return __copy_from_user_inatomic(data, hva, len);
  906. }
  907. int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
  908. unsigned long start, int write, struct page **page)
  909. {
  910. int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
  911. if (write)
  912. flags |= FOLL_WRITE;
  913. return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
  914. }
  915. static inline int check_user_page_hwpoison(unsigned long addr)
  916. {
  917. int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
  918. rc = __get_user_pages(current, current->mm, addr, 1,
  919. flags, NULL, NULL, NULL);
  920. return rc == -EHWPOISON;
  921. }
  922. /*
  923. * The atomic path to get the writable pfn which will be stored in @pfn,
  924. * true indicates success, otherwise false is returned.
  925. */
  926. static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
  927. bool write_fault, bool *writable, pfn_t *pfn)
  928. {
  929. struct page *page[1];
  930. int npages;
  931. if (!(async || atomic))
  932. return false;
  933. /*
  934. * Fast pin a writable pfn only if it is a write fault request
  935. * or the caller allows to map a writable pfn for a read fault
  936. * request.
  937. */
  938. if (!(write_fault || writable))
  939. return false;
  940. npages = __get_user_pages_fast(addr, 1, 1, page);
  941. if (npages == 1) {
  942. *pfn = page_to_pfn(page[0]);
  943. if (writable)
  944. *writable = true;
  945. return true;
  946. }
  947. return false;
  948. }
  949. /*
  950. * The slow path to get the pfn of the specified host virtual address,
  951. * 1 indicates success, -errno is returned if error is detected.
  952. */
  953. static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
  954. bool *writable, pfn_t *pfn)
  955. {
  956. struct page *page[1];
  957. int npages = 0;
  958. might_sleep();
  959. if (writable)
  960. *writable = write_fault;
  961. if (async) {
  962. down_read(&current->mm->mmap_sem);
  963. npages = get_user_page_nowait(current, current->mm,
  964. addr, write_fault, page);
  965. up_read(&current->mm->mmap_sem);
  966. } else
  967. npages = get_user_pages_fast(addr, 1, write_fault,
  968. page);
  969. if (npages != 1)
  970. return npages;
  971. /* map read fault as writable if possible */
  972. if (unlikely(!write_fault) && writable) {
  973. struct page *wpage[1];
  974. npages = __get_user_pages_fast(addr, 1, 1, wpage);
  975. if (npages == 1) {
  976. *writable = true;
  977. put_page(page[0]);
  978. page[0] = wpage[0];
  979. }
  980. npages = 1;
  981. }
  982. *pfn = page_to_pfn(page[0]);
  983. return npages;
  984. }
  985. static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
  986. {
  987. if (unlikely(!(vma->vm_flags & VM_READ)))
  988. return false;
  989. if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
  990. return false;
  991. return true;
  992. }
  993. /*
  994. * Pin guest page in memory and return its pfn.
  995. * @addr: host virtual address which maps memory to the guest
  996. * @atomic: whether this function can sleep
  997. * @async: whether this function need to wait IO complete if the
  998. * host page is not in the memory
  999. * @write_fault: whether we should get a writable host page
  1000. * @writable: whether it allows to map a writable host page for !@write_fault
  1001. *
  1002. * The function will map a writable host page for these two cases:
  1003. * 1): @write_fault = true
  1004. * 2): @write_fault = false && @writable, @writable will tell the caller
  1005. * whether the mapping is writable.
  1006. */
  1007. static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
  1008. bool write_fault, bool *writable)
  1009. {
  1010. struct vm_area_struct *vma;
  1011. pfn_t pfn = 0;
  1012. int npages;
  1013. /* we can do it either atomically or asynchronously, not both */
  1014. BUG_ON(atomic && async);
  1015. if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
  1016. return pfn;
  1017. if (atomic)
  1018. return KVM_PFN_ERR_FAULT;
  1019. npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
  1020. if (npages == 1)
  1021. return pfn;
  1022. down_read(&current->mm->mmap_sem);
  1023. if (npages == -EHWPOISON ||
  1024. (!async && check_user_page_hwpoison(addr))) {
  1025. pfn = KVM_PFN_ERR_HWPOISON;
  1026. goto exit;
  1027. }
  1028. vma = find_vma_intersection(current->mm, addr, addr + 1);
  1029. if (vma == NULL)
  1030. pfn = KVM_PFN_ERR_FAULT;
  1031. else if ((vma->vm_flags & VM_PFNMAP)) {
  1032. pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
  1033. vma->vm_pgoff;
  1034. BUG_ON(!kvm_is_mmio_pfn(pfn));
  1035. } else {
  1036. if (async && vma_is_valid(vma, write_fault))
  1037. *async = true;
  1038. pfn = KVM_PFN_ERR_FAULT;
  1039. }
  1040. exit:
  1041. up_read(&current->mm->mmap_sem);
  1042. return pfn;
  1043. }
  1044. static pfn_t
  1045. __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
  1046. bool *async, bool write_fault, bool *writable)
  1047. {
  1048. unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
  1049. if (addr == KVM_HVA_ERR_RO_BAD)
  1050. return KVM_PFN_ERR_RO_FAULT;
  1051. if (kvm_is_error_hva(addr))
  1052. return KVM_PFN_NOSLOT;
  1053. /* Do not map writable pfn in the readonly memslot. */
  1054. if (writable && memslot_is_readonly(slot)) {
  1055. *writable = false;
  1056. writable = NULL;
  1057. }
  1058. return hva_to_pfn(addr, atomic, async, write_fault,
  1059. writable);
  1060. }
  1061. static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
  1062. bool write_fault, bool *writable)
  1063. {
  1064. struct kvm_memory_slot *slot;
  1065. if (async)
  1066. *async = false;
  1067. slot = gfn_to_memslot(kvm, gfn);
  1068. return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
  1069. writable);
  1070. }
  1071. pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
  1072. {
  1073. return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
  1074. }
  1075. EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
  1076. pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
  1077. bool write_fault, bool *writable)
  1078. {
  1079. return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
  1080. }
  1081. EXPORT_SYMBOL_GPL(gfn_to_pfn_async);
  1082. pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
  1083. {
  1084. return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
  1085. }
  1086. EXPORT_SYMBOL_GPL(gfn_to_pfn);
  1087. pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
  1088. bool *writable)
  1089. {
  1090. return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
  1091. }
  1092. EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
  1093. pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
  1094. {
  1095. return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
  1096. }
  1097. pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
  1098. {
  1099. return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
  1100. }
  1101. EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
  1102. int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
  1103. int nr_pages)
  1104. {
  1105. unsigned long addr;
  1106. gfn_t entry;
  1107. addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
  1108. if (kvm_is_error_hva(addr))
  1109. return -1;
  1110. if (entry < nr_pages)
  1111. return 0;
  1112. return __get_user_pages_fast(addr, nr_pages, 1, pages);
  1113. }
  1114. EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
  1115. static struct page *kvm_pfn_to_page(pfn_t pfn)
  1116. {
  1117. if (is_error_noslot_pfn(pfn))
  1118. return KVM_ERR_PTR_BAD_PAGE;
  1119. if (kvm_is_mmio_pfn(pfn)) {
  1120. WARN_ON(1);
  1121. return KVM_ERR_PTR_BAD_PAGE;
  1122. }
  1123. return pfn_to_page(pfn);
  1124. }
  1125. struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
  1126. {
  1127. pfn_t pfn;
  1128. pfn = gfn_to_pfn(kvm, gfn);
  1129. return kvm_pfn_to_page(pfn);
  1130. }
  1131. EXPORT_SYMBOL_GPL(gfn_to_page);
  1132. void kvm_release_page_clean(struct page *page)
  1133. {
  1134. WARN_ON(is_error_page(page));
  1135. kvm_release_pfn_clean(page_to_pfn(page));
  1136. }
  1137. EXPORT_SYMBOL_GPL(kvm_release_page_clean);
  1138. void kvm_release_pfn_clean(pfn_t pfn)
  1139. {
  1140. if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn))
  1141. put_page(pfn_to_page(pfn));
  1142. }
  1143. EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
  1144. void kvm_release_page_dirty(struct page *page)
  1145. {
  1146. WARN_ON(is_error_page(page));
  1147. kvm_release_pfn_dirty(page_to_pfn(page));
  1148. }
  1149. EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
  1150. void kvm_release_pfn_dirty(pfn_t pfn)
  1151. {
  1152. kvm_set_pfn_dirty(pfn);
  1153. kvm_release_pfn_clean(pfn);
  1154. }
  1155. EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
  1156. void kvm_set_page_dirty(struct page *page)
  1157. {
  1158. kvm_set_pfn_dirty(page_to_pfn(page));
  1159. }
  1160. EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
  1161. void kvm_set_pfn_dirty(pfn_t pfn)
  1162. {
  1163. if (!kvm_is_mmio_pfn(pfn)) {
  1164. struct page *page = pfn_to_page(pfn);
  1165. if (!PageReserved(page))
  1166. SetPageDirty(page);
  1167. }
  1168. }
  1169. EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
  1170. void kvm_set_pfn_accessed(pfn_t pfn)
  1171. {
  1172. if (!kvm_is_mmio_pfn(pfn))
  1173. mark_page_accessed(pfn_to_page(pfn));
  1174. }
  1175. EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
  1176. void kvm_get_pfn(pfn_t pfn)
  1177. {
  1178. if (!kvm_is_mmio_pfn(pfn))
  1179. get_page(pfn_to_page(pfn));
  1180. }
  1181. EXPORT_SYMBOL_GPL(kvm_get_pfn);
  1182. static int next_segment(unsigned long len, int offset)
  1183. {
  1184. if (len > PAGE_SIZE - offset)
  1185. return PAGE_SIZE - offset;
  1186. else
  1187. return len;
  1188. }
  1189. int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
  1190. int len)
  1191. {
  1192. int r;
  1193. unsigned long addr;
  1194. addr = gfn_to_hva_read(kvm, gfn);
  1195. if (kvm_is_error_hva(addr))
  1196. return -EFAULT;
  1197. r = kvm_read_hva(data, (void __user *)addr + offset, len);
  1198. if (r)
  1199. return -EFAULT;
  1200. return 0;
  1201. }
  1202. EXPORT_SYMBOL_GPL(kvm_read_guest_page);
  1203. int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
  1204. {
  1205. gfn_t gfn = gpa >> PAGE_SHIFT;
  1206. int seg;
  1207. int offset = offset_in_page(gpa);
  1208. int ret;
  1209. while ((seg = next_segment(len, offset)) != 0) {
  1210. ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
  1211. if (ret < 0)
  1212. return ret;
  1213. offset = 0;
  1214. len -= seg;
  1215. data += seg;
  1216. ++gfn;
  1217. }
  1218. return 0;
  1219. }
  1220. EXPORT_SYMBOL_GPL(kvm_read_guest);
  1221. int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
  1222. unsigned long len)
  1223. {
  1224. int r;
  1225. unsigned long addr;
  1226. gfn_t gfn = gpa >> PAGE_SHIFT;
  1227. int offset = offset_in_page(gpa);
  1228. addr = gfn_to_hva_read(kvm, gfn);
  1229. if (kvm_is_error_hva(addr))
  1230. return -EFAULT;
  1231. pagefault_disable();
  1232. r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
  1233. pagefault_enable();
  1234. if (r)
  1235. return -EFAULT;
  1236. return 0;
  1237. }
  1238. EXPORT_SYMBOL(kvm_read_guest_atomic);
  1239. int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
  1240. int offset, int len)
  1241. {
  1242. int r;
  1243. unsigned long addr;
  1244. addr = gfn_to_hva(kvm, gfn);
  1245. if (kvm_is_error_hva(addr))
  1246. return -EFAULT;
  1247. r = __copy_to_user((void __user *)addr + offset, data, len);
  1248. if (r)
  1249. return -EFAULT;
  1250. mark_page_dirty(kvm, gfn);
  1251. return 0;
  1252. }
  1253. EXPORT_SYMBOL_GPL(kvm_write_guest_page);
  1254. int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
  1255. unsigned long len)
  1256. {
  1257. gfn_t gfn = gpa >> PAGE_SHIFT;
  1258. int seg;
  1259. int offset = offset_in_page(gpa);
  1260. int ret;
  1261. while ((seg = next_segment(len, offset)) != 0) {
  1262. ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
  1263. if (ret < 0)
  1264. return ret;
  1265. offset = 0;
  1266. len -= seg;
  1267. data += seg;
  1268. ++gfn;
  1269. }
  1270. return 0;
  1271. }
  1272. int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1273. gpa_t gpa)
  1274. {
  1275. struct kvm_memslots *slots = kvm_memslots(kvm);
  1276. int offset = offset_in_page(gpa);
  1277. gfn_t gfn = gpa >> PAGE_SHIFT;
  1278. ghc->gpa = gpa;
  1279. ghc->generation = slots->generation;
  1280. ghc->memslot = gfn_to_memslot(kvm, gfn);
  1281. ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL);
  1282. if (!kvm_is_error_hva(ghc->hva))
  1283. ghc->hva += offset;
  1284. else
  1285. return -EFAULT;
  1286. return 0;
  1287. }
  1288. EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
  1289. int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1290. void *data, unsigned long len)
  1291. {
  1292. struct kvm_memslots *slots = kvm_memslots(kvm);
  1293. int r;
  1294. if (slots->generation != ghc->generation)
  1295. kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
  1296. if (kvm_is_error_hva(ghc->hva))
  1297. return -EFAULT;
  1298. r = __copy_to_user((void __user *)ghc->hva, data, len);
  1299. if (r)
  1300. return -EFAULT;
  1301. mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);
  1302. return 0;
  1303. }
  1304. EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
  1305. int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  1306. void *data, unsigned long len)
  1307. {
  1308. struct kvm_memslots *slots = kvm_memslots(kvm);
  1309. int r;
  1310. if (slots->generation != ghc->generation)
  1311. kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
  1312. if (kvm_is_error_hva(ghc->hva))
  1313. return -EFAULT;
  1314. r = __copy_from_user(data, (void __user *)ghc->hva, len);
  1315. if (r)
  1316. return -EFAULT;
  1317. return 0;
  1318. }
  1319. EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
  1320. int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
  1321. {
  1322. return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
  1323. offset, len);
  1324. }
  1325. EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
  1326. int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
  1327. {
  1328. gfn_t gfn = gpa >> PAGE_SHIFT;
  1329. int seg;
  1330. int offset = offset_in_page(gpa);
  1331. int ret;
  1332. while ((seg = next_segment(len, offset)) != 0) {
  1333. ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
  1334. if (ret < 0)
  1335. return ret;
  1336. offset = 0;
  1337. len -= seg;
  1338. ++gfn;
  1339. }
  1340. return 0;
  1341. }
  1342. EXPORT_SYMBOL_GPL(kvm_clear_guest);
  1343. void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
  1344. gfn_t gfn)
  1345. {
  1346. if (memslot && memslot->dirty_bitmap) {
  1347. unsigned long rel_gfn = gfn - memslot->base_gfn;
  1348. set_bit_le(rel_gfn, memslot->dirty_bitmap);
  1349. }
  1350. }
  1351. void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
  1352. {
  1353. struct kvm_memory_slot *memslot;
  1354. memslot = gfn_to_memslot(kvm, gfn);
  1355. mark_page_dirty_in_slot(kvm, memslot, gfn);
  1356. }
  1357. /*
  1358. * The vCPU has executed a HLT instruction with in-kernel mode enabled.
  1359. */
  1360. void kvm_vcpu_block(struct kvm_vcpu *vcpu)
  1361. {
  1362. DEFINE_WAIT(wait);
  1363. for (;;) {
  1364. prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
  1365. if (kvm_arch_vcpu_runnable(vcpu)) {
  1366. kvm_make_request(KVM_REQ_UNHALT, vcpu);
  1367. break;
  1368. }
  1369. if (kvm_cpu_has_pending_timer(vcpu))
  1370. break;
  1371. if (signal_pending(current))
  1372. break;
  1373. schedule();
  1374. }
  1375. finish_wait(&vcpu->wq, &wait);
  1376. }
  1377. #ifndef CONFIG_S390
  1378. /*
  1379. * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
  1380. */
  1381. void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
  1382. {
  1383. int me;
  1384. int cpu = vcpu->cpu;
  1385. wait_queue_head_t *wqp;
  1386. wqp = kvm_arch_vcpu_wq(vcpu);
  1387. if (waitqueue_active(wqp)) {
  1388. wake_up_interruptible(wqp);
  1389. ++vcpu->stat.halt_wakeup;
  1390. }
  1391. me = get_cpu();
  1392. if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
  1393. if (kvm_arch_vcpu_should_kick(vcpu))
  1394. smp_send_reschedule(cpu);
  1395. put_cpu();
  1396. }
  1397. #endif /* !CONFIG_S390 */
  1398. void kvm_resched(struct kvm_vcpu *vcpu)
  1399. {
  1400. if (!need_resched())
  1401. return;
  1402. cond_resched();
  1403. }
  1404. EXPORT_SYMBOL_GPL(kvm_resched);
  1405. bool kvm_vcpu_yield_to(struct kvm_vcpu *target)
  1406. {
  1407. struct pid *pid;
  1408. struct task_struct *task = NULL;
  1409. rcu_read_lock();
  1410. pid = rcu_dereference(target->pid);
  1411. if (pid)
  1412. task = get_pid_task(target->pid, PIDTYPE_PID);
  1413. rcu_read_unlock();
  1414. if (!task)
  1415. return false;
  1416. if (task->flags & PF_VCPU) {
  1417. put_task_struct(task);
  1418. return false;
  1419. }
  1420. if (yield_to(task, 1)) {
  1421. put_task_struct(task);
  1422. return true;
  1423. }
  1424. put_task_struct(task);
  1425. return false;
  1426. }
  1427. EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
  1428. #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
  1429. /*
  1430. * Helper that checks whether a VCPU is eligible for directed yield.
  1431. * Most eligible candidate to yield is decided by following heuristics:
  1432. *
  1433. * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
  1434. * (preempted lock holder), indicated by @in_spin_loop.
  1435. * Set at the beiginning and cleared at the end of interception/PLE handler.
  1436. *
  1437. * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
  1438. * chance last time (mostly it has become eligible now since we have probably
  1439. * yielded to lockholder in last iteration. This is done by toggling
  1440. * @dy_eligible each time a VCPU checked for eligibility.)
  1441. *
  1442. * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
  1443. * to preempted lock-holder could result in wrong VCPU selection and CPU
  1444. * burning. Giving priority for a potential lock-holder increases lock
  1445. * progress.
  1446. *
  1447. * Since algorithm is based on heuristics, accessing another VCPU data without
  1448. * locking does not harm. It may result in trying to yield to same VCPU, fail
  1449. * and continue with next VCPU and so on.
  1450. */
  1451. bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
  1452. {
  1453. bool eligible;
  1454. eligible = !vcpu->spin_loop.in_spin_loop ||
  1455. (vcpu->spin_loop.in_spin_loop &&
  1456. vcpu->spin_loop.dy_eligible);
  1457. if (vcpu->spin_loop.in_spin_loop)
  1458. kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
  1459. return eligible;
  1460. }
  1461. #endif
  1462. void kvm_vcpu_on_spin(struct kvm_vcpu *me)
  1463. {
  1464. struct kvm *kvm = me->kvm;
  1465. struct kvm_vcpu *vcpu;
  1466. int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
  1467. int yielded = 0;
  1468. int pass;
  1469. int i;
  1470. kvm_vcpu_set_in_spin_loop(me, true);
  1471. /*
  1472. * We boost the priority of a VCPU that is runnable but not
  1473. * currently running, because it got preempted by something
  1474. * else and called schedule in __vcpu_run. Hopefully that
  1475. * VCPU is holding the lock that we need and will release it.
  1476. * We approximate round-robin by starting at the last boosted VCPU.
  1477. */
  1478. for (pass = 0; pass < 2 && !yielded; pass++) {
  1479. kvm_for_each_vcpu(i, vcpu, kvm) {
  1480. if (!pass && i <= last_boosted_vcpu) {
  1481. i = last_boosted_vcpu;
  1482. continue;
  1483. } else if (pass && i > last_boosted_vcpu)
  1484. break;
  1485. if (vcpu == me)
  1486. continue;
  1487. if (waitqueue_active(&vcpu->wq))
  1488. continue;
  1489. if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
  1490. continue;
  1491. if (kvm_vcpu_yield_to(vcpu)) {
  1492. kvm->last_boosted_vcpu = i;
  1493. yielded = 1;
  1494. break;
  1495. }
  1496. }
  1497. }
  1498. kvm_vcpu_set_in_spin_loop(me, false);
  1499. /* Ensure vcpu is not eligible during next spinloop */
  1500. kvm_vcpu_set_dy_eligible(me, false);
  1501. }
  1502. EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
  1503. static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1504. {
  1505. struct kvm_vcpu *vcpu = vma->vm_file->private_data;
  1506. struct page *page;
  1507. if (vmf->pgoff == 0)
  1508. page = virt_to_page(vcpu->run);
  1509. #ifdef CONFIG_X86
  1510. else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
  1511. page = virt_to_page(vcpu->arch.pio_data);
  1512. #endif
  1513. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  1514. else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
  1515. page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
  1516. #endif
  1517. else
  1518. return kvm_arch_vcpu_fault(vcpu, vmf);
  1519. get_page(page);
  1520. vmf->page = page;
  1521. return 0;
  1522. }
  1523. static const struct vm_operations_struct kvm_vcpu_vm_ops = {
  1524. .fault = kvm_vcpu_fault,
  1525. };
  1526. static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
  1527. {
  1528. vma->vm_ops = &kvm_vcpu_vm_ops;
  1529. return 0;
  1530. }
  1531. static int kvm_vcpu_release(struct inode *inode, struct file *filp)
  1532. {
  1533. struct kvm_vcpu *vcpu = filp->private_data;
  1534. kvm_put_kvm(vcpu->kvm);
  1535. return 0;
  1536. }
  1537. static struct file_operations kvm_vcpu_fops = {
  1538. .release = kvm_vcpu_release,
  1539. .unlocked_ioctl = kvm_vcpu_ioctl,
  1540. #ifdef CONFIG_COMPAT
  1541. .compat_ioctl = kvm_vcpu_compat_ioctl,
  1542. #endif
  1543. .mmap = kvm_vcpu_mmap,
  1544. .llseek = noop_llseek,
  1545. };
  1546. /*
  1547. * Allocates an inode for the vcpu.
  1548. */
  1549. static int create_vcpu_fd(struct kvm_vcpu *vcpu)
  1550. {
  1551. return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
  1552. }
  1553. /*
  1554. * Creates some virtual cpus. Good luck creating more than one.
  1555. */
  1556. static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
  1557. {
  1558. int r;
  1559. struct kvm_vcpu *vcpu, *v;
  1560. vcpu = kvm_arch_vcpu_create(kvm, id);
  1561. if (IS_ERR(vcpu))
  1562. return PTR_ERR(vcpu);
  1563. preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
  1564. r = kvm_arch_vcpu_setup(vcpu);
  1565. if (r)
  1566. goto vcpu_destroy;
  1567. mutex_lock(&kvm->lock);
  1568. if (!kvm_vcpu_compatible(vcpu)) {
  1569. r = -EINVAL;
  1570. goto unlock_vcpu_destroy;
  1571. }
  1572. if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
  1573. r = -EINVAL;
  1574. goto unlock_vcpu_destroy;
  1575. }
  1576. kvm_for_each_vcpu(r, v, kvm)
  1577. if (v->vcpu_id == id) {
  1578. r = -EEXIST;
  1579. goto unlock_vcpu_destroy;
  1580. }
  1581. BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
  1582. /* Now it's all set up, let userspace reach it */
  1583. kvm_get_kvm(kvm);
  1584. r = create_vcpu_fd(vcpu);
  1585. if (r < 0) {
  1586. kvm_put_kvm(kvm);
  1587. goto unlock_vcpu_destroy;
  1588. }
  1589. kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
  1590. smp_wmb();
  1591. atomic_inc(&kvm->online_vcpus);
  1592. mutex_unlock(&kvm->lock);
  1593. kvm_arch_vcpu_postcreate(vcpu);
  1594. return r;
  1595. unlock_vcpu_destroy:
  1596. mutex_unlock(&kvm->lock);
  1597. vcpu_destroy:
  1598. kvm_arch_vcpu_destroy(vcpu);
  1599. return r;
  1600. }
  1601. static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
  1602. {
  1603. if (sigset) {
  1604. sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
  1605. vcpu->sigset_active = 1;
  1606. vcpu->sigset = *sigset;
  1607. } else
  1608. vcpu->sigset_active = 0;
  1609. return 0;
  1610. }
  1611. static long kvm_vcpu_ioctl(struct file *filp,
  1612. unsigned int ioctl, unsigned long arg)
  1613. {
  1614. struct kvm_vcpu *vcpu = filp->private_data;
  1615. void __user *argp = (void __user *)arg;
  1616. int r;
  1617. struct kvm_fpu *fpu = NULL;
  1618. struct kvm_sregs *kvm_sregs = NULL;
  1619. if (vcpu->kvm->mm != current->mm)
  1620. return -EIO;
  1621. #if defined(CONFIG_S390) || defined(CONFIG_PPC)
  1622. /*
  1623. * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
  1624. * so vcpu_load() would break it.
  1625. */
  1626. if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
  1627. return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  1628. #endif
  1629. r = vcpu_load(vcpu);
  1630. if (r)
  1631. return r;
  1632. switch (ioctl) {
  1633. case KVM_RUN:
  1634. r = -EINVAL;
  1635. if (arg)
  1636. goto out;
  1637. r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
  1638. trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
  1639. break;
  1640. case KVM_GET_REGS: {
  1641. struct kvm_regs *kvm_regs;
  1642. r = -ENOMEM;
  1643. kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
  1644. if (!kvm_regs)
  1645. goto out;
  1646. r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
  1647. if (r)
  1648. goto out_free1;
  1649. r = -EFAULT;
  1650. if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
  1651. goto out_free1;
  1652. r = 0;
  1653. out_free1:
  1654. kfree(kvm_regs);
  1655. break;
  1656. }
  1657. case KVM_SET_REGS: {
  1658. struct kvm_regs *kvm_regs;
  1659. r = -ENOMEM;
  1660. kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
  1661. if (IS_ERR(kvm_regs)) {
  1662. r = PTR_ERR(kvm_regs);
  1663. goto out;
  1664. }
  1665. r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
  1666. kfree(kvm_regs);
  1667. break;
  1668. }
  1669. case KVM_GET_SREGS: {
  1670. kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
  1671. r = -ENOMEM;
  1672. if (!kvm_sregs)
  1673. goto out;
  1674. r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
  1675. if (r)
  1676. goto out;
  1677. r = -EFAULT;
  1678. if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
  1679. goto out;
  1680. r = 0;
  1681. break;
  1682. }
  1683. case KVM_SET_SREGS: {
  1684. kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
  1685. if (IS_ERR(kvm_sregs)) {
  1686. r = PTR_ERR(kvm_sregs);
  1687. kvm_sregs = NULL;
  1688. goto out;
  1689. }
  1690. r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
  1691. break;
  1692. }
  1693. case KVM_GET_MP_STATE: {
  1694. struct kvm_mp_state mp_state;
  1695. r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
  1696. if (r)
  1697. goto out;
  1698. r = -EFAULT;
  1699. if (copy_to_user(argp, &mp_state, sizeof mp_state))
  1700. goto out;
  1701. r = 0;
  1702. break;
  1703. }
  1704. case KVM_SET_MP_STATE: {
  1705. struct kvm_mp_state mp_state;
  1706. r = -EFAULT;
  1707. if (copy_from_user(&mp_state, argp, sizeof mp_state))
  1708. goto out;
  1709. r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
  1710. break;
  1711. }
  1712. case KVM_TRANSLATE: {
  1713. struct kvm_translation tr;
  1714. r = -EFAULT;
  1715. if (copy_from_user(&tr, argp, sizeof tr))
  1716. goto out;
  1717. r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
  1718. if (r)
  1719. goto out;
  1720. r = -EFAULT;
  1721. if (copy_to_user(argp, &tr, sizeof tr))
  1722. goto out;
  1723. r = 0;
  1724. break;
  1725. }
  1726. case KVM_SET_GUEST_DEBUG: {
  1727. struct kvm_guest_debug dbg;
  1728. r = -EFAULT;
  1729. if (copy_from_user(&dbg, argp, sizeof dbg))
  1730. goto out;
  1731. r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
  1732. break;
  1733. }
  1734. case KVM_SET_SIGNAL_MASK: {
  1735. struct kvm_signal_mask __user *sigmask_arg = argp;
  1736. struct kvm_signal_mask kvm_sigmask;
  1737. sigset_t sigset, *p;
  1738. p = NULL;
  1739. if (argp) {
  1740. r = -EFAULT;
  1741. if (copy_from_user(&kvm_sigmask, argp,
  1742. sizeof kvm_sigmask))
  1743. goto out;
  1744. r = -EINVAL;
  1745. if (kvm_sigmask.len != sizeof sigset)
  1746. goto out;
  1747. r = -EFAULT;
  1748. if (copy_from_user(&sigset, sigmask_arg->sigset,
  1749. sizeof sigset))
  1750. goto out;
  1751. p = &sigset;
  1752. }
  1753. r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
  1754. break;
  1755. }
  1756. case KVM_GET_FPU: {
  1757. fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
  1758. r = -ENOMEM;
  1759. if (!fpu)
  1760. goto out;
  1761. r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
  1762. if (r)
  1763. goto out;
  1764. r = -EFAULT;
  1765. if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
  1766. goto out;
  1767. r = 0;
  1768. break;
  1769. }
  1770. case KVM_SET_FPU: {
  1771. fpu = memdup_user(argp, sizeof(*fpu));
  1772. if (IS_ERR(fpu)) {
  1773. r = PTR_ERR(fpu);
  1774. fpu = NULL;
  1775. goto out;
  1776. }
  1777. r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
  1778. break;
  1779. }
  1780. default:
  1781. r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
  1782. }
  1783. out:
  1784. vcpu_put(vcpu);
  1785. kfree(fpu);
  1786. kfree(kvm_sregs);
  1787. return r;
  1788. }
  1789. #ifdef CONFIG_COMPAT
  1790. static long kvm_vcpu_compat_ioctl(struct file *filp,
  1791. unsigned int ioctl, unsigned long arg)
  1792. {
  1793. struct kvm_vcpu *vcpu = filp->private_data;
  1794. void __user *argp = compat_ptr(arg);
  1795. int r;
  1796. if (vcpu->kvm->mm != current->mm)
  1797. return -EIO;
  1798. switch (ioctl) {
  1799. case KVM_SET_SIGNAL_MASK: {
  1800. struct kvm_signal_mask __user *sigmask_arg = argp;
  1801. struct kvm_signal_mask kvm_sigmask;
  1802. compat_sigset_t csigset;
  1803. sigset_t sigset;
  1804. if (argp) {
  1805. r = -EFAULT;
  1806. if (copy_from_user(&kvm_sigmask, argp,
  1807. sizeof kvm_sigmask))
  1808. goto out;
  1809. r = -EINVAL;
  1810. if (kvm_sigmask.len != sizeof csigset)
  1811. goto out;
  1812. r = -EFAULT;
  1813. if (copy_from_user(&csigset, sigmask_arg->sigset,
  1814. sizeof csigset))
  1815. goto out;
  1816. sigset_from_compat(&sigset, &csigset);
  1817. r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
  1818. } else
  1819. r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
  1820. break;
  1821. }
  1822. default:
  1823. r = kvm_vcpu_ioctl(filp, ioctl, arg);
  1824. }
  1825. out:
  1826. return r;
  1827. }
  1828. #endif
  1829. static long kvm_vm_ioctl(struct file *filp,
  1830. unsigned int ioctl, unsigned long arg)
  1831. {
  1832. struct kvm *kvm = filp->private_data;
  1833. void __user *argp = (void __user *)arg;
  1834. int r;
  1835. if (kvm->mm != current->mm)
  1836. return -EIO;
  1837. switch (ioctl) {
  1838. case KVM_CREATE_VCPU:
  1839. r = kvm_vm_ioctl_create_vcpu(kvm, arg);
  1840. break;
  1841. case KVM_SET_USER_MEMORY_REGION: {
  1842. struct kvm_userspace_memory_region kvm_userspace_mem;
  1843. r = -EFAULT;
  1844. if (copy_from_user(&kvm_userspace_mem, argp,
  1845. sizeof kvm_userspace_mem))
  1846. goto out;
  1847. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, true);
  1848. break;
  1849. }
  1850. case KVM_GET_DIRTY_LOG: {
  1851. struct kvm_dirty_log log;
  1852. r = -EFAULT;
  1853. if (copy_from_user(&log, argp, sizeof log))
  1854. goto out;
  1855. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  1856. break;
  1857. }
  1858. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  1859. case KVM_REGISTER_COALESCED_MMIO: {
  1860. struct kvm_coalesced_mmio_zone zone;
  1861. r = -EFAULT;
  1862. if (copy_from_user(&zone, argp, sizeof zone))
  1863. goto out;
  1864. r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
  1865. break;
  1866. }
  1867. case KVM_UNREGISTER_COALESCED_MMIO: {
  1868. struct kvm_coalesced_mmio_zone zone;
  1869. r = -EFAULT;
  1870. if (copy_from_user(&zone, argp, sizeof zone))
  1871. goto out;
  1872. r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
  1873. break;
  1874. }
  1875. #endif
  1876. case KVM_IRQFD: {
  1877. struct kvm_irqfd data;
  1878. r = -EFAULT;
  1879. if (copy_from_user(&data, argp, sizeof data))
  1880. goto out;
  1881. r = kvm_irqfd(kvm, &data);
  1882. break;
  1883. }
  1884. case KVM_IOEVENTFD: {
  1885. struct kvm_ioeventfd data;
  1886. r = -EFAULT;
  1887. if (copy_from_user(&data, argp, sizeof data))
  1888. goto out;
  1889. r = kvm_ioeventfd(kvm, &data);
  1890. break;
  1891. }
  1892. #ifdef CONFIG_KVM_APIC_ARCHITECTURE
  1893. case KVM_SET_BOOT_CPU_ID:
  1894. r = 0;
  1895. mutex_lock(&kvm->lock);
  1896. if (atomic_read(&kvm->online_vcpus) != 0)
  1897. r = -EBUSY;
  1898. else
  1899. kvm->bsp_vcpu_id = arg;
  1900. mutex_unlock(&kvm->lock);
  1901. break;
  1902. #endif
  1903. #ifdef CONFIG_HAVE_KVM_MSI
  1904. case KVM_SIGNAL_MSI: {
  1905. struct kvm_msi msi;
  1906. r = -EFAULT;
  1907. if (copy_from_user(&msi, argp, sizeof msi))
  1908. goto out;
  1909. r = kvm_send_userspace_msi(kvm, &msi);
  1910. break;
  1911. }
  1912. #endif
  1913. #ifdef __KVM_HAVE_IRQ_LINE
  1914. case KVM_IRQ_LINE_STATUS:
  1915. case KVM_IRQ_LINE: {
  1916. struct kvm_irq_level irq_event;
  1917. r = -EFAULT;
  1918. if (copy_from_user(&irq_event, argp, sizeof irq_event))
  1919. goto out;
  1920. r = kvm_vm_ioctl_irq_line(kvm, &irq_event);
  1921. if (r)
  1922. goto out;
  1923. r = -EFAULT;
  1924. if (ioctl == KVM_IRQ_LINE_STATUS) {
  1925. if (copy_to_user(argp, &irq_event, sizeof irq_event))
  1926. goto out;
  1927. }
  1928. r = 0;
  1929. break;
  1930. }
  1931. #endif
  1932. default:
  1933. r = kvm_arch_vm_ioctl(filp, ioctl, arg);
  1934. if (r == -ENOTTY)
  1935. r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
  1936. }
  1937. out:
  1938. return r;
  1939. }
  1940. #ifdef CONFIG_COMPAT
  1941. struct compat_kvm_dirty_log {
  1942. __u32 slot;
  1943. __u32 padding1;
  1944. union {
  1945. compat_uptr_t dirty_bitmap; /* one bit per page */
  1946. __u64 padding2;
  1947. };
  1948. };
  1949. static long kvm_vm_compat_ioctl(struct file *filp,
  1950. unsigned int ioctl, unsigned long arg)
  1951. {
  1952. struct kvm *kvm = filp->private_data;
  1953. int r;
  1954. if (kvm->mm != current->mm)
  1955. return -EIO;
  1956. switch (ioctl) {
  1957. case KVM_GET_DIRTY_LOG: {
  1958. struct compat_kvm_dirty_log compat_log;
  1959. struct kvm_dirty_log log;
  1960. r = -EFAULT;
  1961. if (copy_from_user(&compat_log, (void __user *)arg,
  1962. sizeof(compat_log)))
  1963. goto out;
  1964. log.slot = compat_log.slot;
  1965. log.padding1 = compat_log.padding1;
  1966. log.padding2 = compat_log.padding2;
  1967. log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
  1968. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  1969. break;
  1970. }
  1971. default:
  1972. r = kvm_vm_ioctl(filp, ioctl, arg);
  1973. }
  1974. out:
  1975. return r;
  1976. }
  1977. #endif
  1978. static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1979. {
  1980. struct page *page[1];
  1981. unsigned long addr;
  1982. int npages;
  1983. gfn_t gfn = vmf->pgoff;
  1984. struct kvm *kvm = vma->vm_file->private_data;
  1985. addr = gfn_to_hva(kvm, gfn);
  1986. if (kvm_is_error_hva(addr))
  1987. return VM_FAULT_SIGBUS;
  1988. npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
  1989. NULL);
  1990. if (unlikely(npages != 1))
  1991. return VM_FAULT_SIGBUS;
  1992. vmf->page = page[0];
  1993. return 0;
  1994. }
  1995. static const struct vm_operations_struct kvm_vm_vm_ops = {
  1996. .fault = kvm_vm_fault,
  1997. };
  1998. static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
  1999. {
  2000. vma->vm_ops = &kvm_vm_vm_ops;
  2001. return 0;
  2002. }
  2003. static struct file_operations kvm_vm_fops = {
  2004. .release = kvm_vm_release,
  2005. .unlocked_ioctl = kvm_vm_ioctl,
  2006. #ifdef CONFIG_COMPAT
  2007. .compat_ioctl = kvm_vm_compat_ioctl,
  2008. #endif
  2009. .mmap = kvm_vm_mmap,
  2010. .llseek = noop_llseek,
  2011. };
  2012. static int kvm_dev_ioctl_create_vm(unsigned long type)
  2013. {
  2014. int r;
  2015. struct kvm *kvm;
  2016. kvm = kvm_create_vm(type);
  2017. if (IS_ERR(kvm))
  2018. return PTR_ERR(kvm);
  2019. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  2020. r = kvm_coalesced_mmio_init(kvm);
  2021. if (r < 0) {
  2022. kvm_put_kvm(kvm);
  2023. return r;
  2024. }
  2025. #endif
  2026. r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
  2027. if (r < 0)
  2028. kvm_put_kvm(kvm);
  2029. return r;
  2030. }
  2031. static long kvm_dev_ioctl_check_extension_generic(long arg)
  2032. {
  2033. switch (arg) {
  2034. case KVM_CAP_USER_MEMORY:
  2035. case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
  2036. case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
  2037. #ifdef CONFIG_KVM_APIC_ARCHITECTURE
  2038. case KVM_CAP_SET_BOOT_CPU_ID:
  2039. #endif
  2040. case KVM_CAP_INTERNAL_ERROR_DATA:
  2041. #ifdef CONFIG_HAVE_KVM_MSI
  2042. case KVM_CAP_SIGNAL_MSI:
  2043. #endif
  2044. return 1;
  2045. #ifdef KVM_CAP_IRQ_ROUTING
  2046. case KVM_CAP_IRQ_ROUTING:
  2047. return KVM_MAX_IRQ_ROUTES;
  2048. #endif
  2049. default:
  2050. break;
  2051. }
  2052. return kvm_dev_ioctl_check_extension(arg);
  2053. }
  2054. static long kvm_dev_ioctl(struct file *filp,
  2055. unsigned int ioctl, unsigned long arg)
  2056. {
  2057. long r = -EINVAL;
  2058. switch (ioctl) {
  2059. case KVM_GET_API_VERSION:
  2060. r = -EINVAL;
  2061. if (arg)
  2062. goto out;
  2063. r = KVM_API_VERSION;
  2064. break;
  2065. case KVM_CREATE_VM:
  2066. r = kvm_dev_ioctl_create_vm(arg);
  2067. break;
  2068. case KVM_CHECK_EXTENSION:
  2069. r = kvm_dev_ioctl_check_extension_generic(arg);
  2070. break;
  2071. case KVM_GET_VCPU_MMAP_SIZE:
  2072. r = -EINVAL;
  2073. if (arg)
  2074. goto out;
  2075. r = PAGE_SIZE; /* struct kvm_run */
  2076. #ifdef CONFIG_X86
  2077. r += PAGE_SIZE; /* pio data page */
  2078. #endif
  2079. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  2080. r += PAGE_SIZE; /* coalesced mmio ring page */
  2081. #endif
  2082. break;
  2083. case KVM_TRACE_ENABLE:
  2084. case KVM_TRACE_PAUSE:
  2085. case KVM_TRACE_DISABLE:
  2086. r = -EOPNOTSUPP;
  2087. break;
  2088. default:
  2089. return kvm_arch_dev_ioctl(filp, ioctl, arg);
  2090. }
  2091. out:
  2092. return r;
  2093. }
  2094. static struct file_operations kvm_chardev_ops = {
  2095. .unlocked_ioctl = kvm_dev_ioctl,
  2096. .compat_ioctl = kvm_dev_ioctl,
  2097. .llseek = noop_llseek,
  2098. };
  2099. static struct miscdevice kvm_dev = {
  2100. KVM_MINOR,
  2101. "kvm",
  2102. &kvm_chardev_ops,
  2103. };
  2104. static void hardware_enable_nolock(void *junk)
  2105. {
  2106. int cpu = raw_smp_processor_id();
  2107. int r;
  2108. if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
  2109. return;
  2110. cpumask_set_cpu(cpu, cpus_hardware_enabled);
  2111. r = kvm_arch_hardware_enable(NULL);
  2112. if (r) {
  2113. cpumask_clear_cpu(cpu, cpus_hardware_enabled);
  2114. atomic_inc(&hardware_enable_failed);
  2115. printk(KERN_INFO "kvm: enabling virtualization on "
  2116. "CPU%d failed\n", cpu);
  2117. }
  2118. }
  2119. static void hardware_enable(void *junk)
  2120. {
  2121. raw_spin_lock(&kvm_lock);
  2122. hardware_enable_nolock(junk);
  2123. raw_spin_unlock(&kvm_lock);
  2124. }
  2125. static void hardware_disable_nolock(void *junk)
  2126. {
  2127. int cpu = raw_smp_processor_id();
  2128. if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
  2129. return;
  2130. cpumask_clear_cpu(cpu, cpus_hardware_enabled);
  2131. kvm_arch_hardware_disable(NULL);
  2132. }
  2133. static void hardware_disable(void *junk)
  2134. {
  2135. raw_spin_lock(&kvm_lock);
  2136. hardware_disable_nolock(junk);
  2137. raw_spin_unlock(&kvm_lock);
  2138. }
  2139. static void hardware_disable_all_nolock(void)
  2140. {
  2141. BUG_ON(!kvm_usage_count);
  2142. kvm_usage_count--;
  2143. if (!kvm_usage_count)
  2144. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2145. }
  2146. static void hardware_disable_all(void)
  2147. {
  2148. raw_spin_lock(&kvm_lock);
  2149. hardware_disable_all_nolock();
  2150. raw_spin_unlock(&kvm_lock);
  2151. }
  2152. static int hardware_enable_all(void)
  2153. {
  2154. int r = 0;
  2155. raw_spin_lock(&kvm_lock);
  2156. kvm_usage_count++;
  2157. if (kvm_usage_count == 1) {
  2158. atomic_set(&hardware_enable_failed, 0);
  2159. on_each_cpu(hardware_enable_nolock, NULL, 1);
  2160. if (atomic_read(&hardware_enable_failed)) {
  2161. hardware_disable_all_nolock();
  2162. r = -EBUSY;
  2163. }
  2164. }
  2165. raw_spin_unlock(&kvm_lock);
  2166. return r;
  2167. }
  2168. static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  2169. void *v)
  2170. {
  2171. int cpu = (long)v;
  2172. if (!kvm_usage_count)
  2173. return NOTIFY_OK;
  2174. val &= ~CPU_TASKS_FROZEN;
  2175. switch (val) {
  2176. case CPU_DYING:
  2177. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  2178. cpu);
  2179. hardware_disable(NULL);
  2180. break;
  2181. case CPU_STARTING:
  2182. printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
  2183. cpu);
  2184. hardware_enable(NULL);
  2185. break;
  2186. }
  2187. return NOTIFY_OK;
  2188. }
  2189. asmlinkage void kvm_spurious_fault(void)
  2190. {
  2191. /* Fault while not rebooting. We want the trace. */
  2192. BUG();
  2193. }
  2194. EXPORT_SYMBOL_GPL(kvm_spurious_fault);
  2195. static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
  2196. void *v)
  2197. {
  2198. /*
  2199. * Some (well, at least mine) BIOSes hang on reboot if
  2200. * in vmx root mode.
  2201. *
  2202. * And Intel TXT required VMX off for all cpu when system shutdown.
  2203. */
  2204. printk(KERN_INFO "kvm: exiting hardware virtualization\n");
  2205. kvm_rebooting = true;
  2206. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2207. return NOTIFY_OK;
  2208. }
  2209. static struct notifier_block kvm_reboot_notifier = {
  2210. .notifier_call = kvm_reboot,
  2211. .priority = 0,
  2212. };
  2213. static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
  2214. {
  2215. int i;
  2216. for (i = 0; i < bus->dev_count; i++) {
  2217. struct kvm_io_device *pos = bus->range[i].dev;
  2218. kvm_iodevice_destructor(pos);
  2219. }
  2220. kfree(bus);
  2221. }
  2222. int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
  2223. {
  2224. const struct kvm_io_range *r1 = p1;
  2225. const struct kvm_io_range *r2 = p2;
  2226. if (r1->addr < r2->addr)
  2227. return -1;
  2228. if (r1->addr + r1->len > r2->addr + r2->len)
  2229. return 1;
  2230. return 0;
  2231. }
  2232. int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
  2233. gpa_t addr, int len)
  2234. {
  2235. bus->range[bus->dev_count++] = (struct kvm_io_range) {
  2236. .addr = addr,
  2237. .len = len,
  2238. .dev = dev,
  2239. };
  2240. sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
  2241. kvm_io_bus_sort_cmp, NULL);
  2242. return 0;
  2243. }
  2244. int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
  2245. gpa_t addr, int len)
  2246. {
  2247. struct kvm_io_range *range, key;
  2248. int off;
  2249. key = (struct kvm_io_range) {
  2250. .addr = addr,
  2251. .len = len,
  2252. };
  2253. range = bsearch(&key, bus->range, bus->dev_count,
  2254. sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
  2255. if (range == NULL)
  2256. return -ENOENT;
  2257. off = range - bus->range;
  2258. while (off > 0 && kvm_io_bus_sort_cmp(&key, &bus->range[off-1]) == 0)
  2259. off--;
  2260. return off;
  2261. }
  2262. /* kvm_io_bus_write - called under kvm->slots_lock */
  2263. int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  2264. int len, const void *val)
  2265. {
  2266. int idx;
  2267. struct kvm_io_bus *bus;
  2268. struct kvm_io_range range;
  2269. range = (struct kvm_io_range) {
  2270. .addr = addr,
  2271. .len = len,
  2272. };
  2273. bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
  2274. idx = kvm_io_bus_get_first_dev(bus, addr, len);
  2275. if (idx < 0)
  2276. return -EOPNOTSUPP;
  2277. while (idx < bus->dev_count &&
  2278. kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
  2279. if (!kvm_iodevice_write(bus->range[idx].dev, addr, len, val))
  2280. return 0;
  2281. idx++;
  2282. }
  2283. return -EOPNOTSUPP;
  2284. }
  2285. /* kvm_io_bus_read - called under kvm->slots_lock */
  2286. int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  2287. int len, void *val)
  2288. {
  2289. int idx;
  2290. struct kvm_io_bus *bus;
  2291. struct kvm_io_range range;
  2292. range = (struct kvm_io_range) {
  2293. .addr = addr,
  2294. .len = len,
  2295. };
  2296. bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
  2297. idx = kvm_io_bus_get_first_dev(bus, addr, len);
  2298. if (idx < 0)
  2299. return -EOPNOTSUPP;
  2300. while (idx < bus->dev_count &&
  2301. kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
  2302. if (!kvm_iodevice_read(bus->range[idx].dev, addr, len, val))
  2303. return 0;
  2304. idx++;
  2305. }
  2306. return -EOPNOTSUPP;
  2307. }
  2308. /* Caller must hold slots_lock. */
  2309. int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  2310. int len, struct kvm_io_device *dev)
  2311. {
  2312. struct kvm_io_bus *new_bus, *bus;
  2313. bus = kvm->buses[bus_idx];
  2314. if (bus->dev_count > NR_IOBUS_DEVS - 1)
  2315. return -ENOSPC;
  2316. new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
  2317. sizeof(struct kvm_io_range)), GFP_KERNEL);
  2318. if (!new_bus)
  2319. return -ENOMEM;
  2320. memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
  2321. sizeof(struct kvm_io_range)));
  2322. kvm_io_bus_insert_dev(new_bus, dev, addr, len);
  2323. rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
  2324. synchronize_srcu_expedited(&kvm->srcu);
  2325. kfree(bus);
  2326. return 0;
  2327. }
  2328. /* Caller must hold slots_lock. */
  2329. int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
  2330. struct kvm_io_device *dev)
  2331. {
  2332. int i, r;
  2333. struct kvm_io_bus *new_bus, *bus;
  2334. bus = kvm->buses[bus_idx];
  2335. r = -ENOENT;
  2336. for (i = 0; i < bus->dev_count; i++)
  2337. if (bus->range[i].dev == dev) {
  2338. r = 0;
  2339. break;
  2340. }
  2341. if (r)
  2342. return r;
  2343. new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
  2344. sizeof(struct kvm_io_range)), GFP_KERNEL);
  2345. if (!new_bus)
  2346. return -ENOMEM;
  2347. memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
  2348. new_bus->dev_count--;
  2349. memcpy(new_bus->range + i, bus->range + i + 1,
  2350. (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
  2351. rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
  2352. synchronize_srcu_expedited(&kvm->srcu);
  2353. kfree(bus);
  2354. return r;
  2355. }
  2356. static struct notifier_block kvm_cpu_notifier = {
  2357. .notifier_call = kvm_cpu_hotplug,
  2358. };
  2359. static int vm_stat_get(void *_offset, u64 *val)
  2360. {
  2361. unsigned offset = (long)_offset;
  2362. struct kvm *kvm;
  2363. *val = 0;
  2364. raw_spin_lock(&kvm_lock);
  2365. list_for_each_entry(kvm, &vm_list, vm_list)
  2366. *val += *(u32 *)((void *)kvm + offset);
  2367. raw_spin_unlock(&kvm_lock);
  2368. return 0;
  2369. }
  2370. DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
  2371. static int vcpu_stat_get(void *_offset, u64 *val)
  2372. {
  2373. unsigned offset = (long)_offset;
  2374. struct kvm *kvm;
  2375. struct kvm_vcpu *vcpu;
  2376. int i;
  2377. *val = 0;
  2378. raw_spin_lock(&kvm_lock);
  2379. list_for_each_entry(kvm, &vm_list, vm_list)
  2380. kvm_for_each_vcpu(i, vcpu, kvm)
  2381. *val += *(u32 *)((void *)vcpu + offset);
  2382. raw_spin_unlock(&kvm_lock);
  2383. return 0;
  2384. }
  2385. DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
  2386. static const struct file_operations *stat_fops[] = {
  2387. [KVM_STAT_VCPU] = &vcpu_stat_fops,
  2388. [KVM_STAT_VM] = &vm_stat_fops,
  2389. };
  2390. static int kvm_init_debug(void)
  2391. {
  2392. int r = -EFAULT;
  2393. struct kvm_stats_debugfs_item *p;
  2394. kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
  2395. if (kvm_debugfs_dir == NULL)
  2396. goto out;
  2397. for (p = debugfs_entries; p->name; ++p) {
  2398. p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
  2399. (void *)(long)p->offset,
  2400. stat_fops[p->kind]);
  2401. if (p->dentry == NULL)
  2402. goto out_dir;
  2403. }
  2404. return 0;
  2405. out_dir:
  2406. debugfs_remove_recursive(kvm_debugfs_dir);
  2407. out:
  2408. return r;
  2409. }
  2410. static void kvm_exit_debug(void)
  2411. {
  2412. struct kvm_stats_debugfs_item *p;
  2413. for (p = debugfs_entries; p->name; ++p)
  2414. debugfs_remove(p->dentry);
  2415. debugfs_remove(kvm_debugfs_dir);
  2416. }
  2417. static int kvm_suspend(void)
  2418. {
  2419. if (kvm_usage_count)
  2420. hardware_disable_nolock(NULL);
  2421. return 0;
  2422. }
  2423. static void kvm_resume(void)
  2424. {
  2425. if (kvm_usage_count) {
  2426. WARN_ON(raw_spin_is_locked(&kvm_lock));
  2427. hardware_enable_nolock(NULL);
  2428. }
  2429. }
  2430. static struct syscore_ops kvm_syscore_ops = {
  2431. .suspend = kvm_suspend,
  2432. .resume = kvm_resume,
  2433. };
  2434. static inline
  2435. struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
  2436. {
  2437. return container_of(pn, struct kvm_vcpu, preempt_notifier);
  2438. }
  2439. static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
  2440. {
  2441. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  2442. kvm_arch_vcpu_load(vcpu, cpu);
  2443. }
  2444. static void kvm_sched_out(struct preempt_notifier *pn,
  2445. struct task_struct *next)
  2446. {
  2447. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  2448. kvm_arch_vcpu_put(vcpu);
  2449. }
  2450. int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
  2451. struct module *module)
  2452. {
  2453. int r;
  2454. int cpu;
  2455. r = kvm_arch_init(opaque);
  2456. if (r)
  2457. goto out_fail;
  2458. if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
  2459. r = -ENOMEM;
  2460. goto out_free_0;
  2461. }
  2462. r = kvm_arch_hardware_setup();
  2463. if (r < 0)
  2464. goto out_free_0a;
  2465. for_each_online_cpu(cpu) {
  2466. smp_call_function_single(cpu,
  2467. kvm_arch_check_processor_compat,
  2468. &r, 1);
  2469. if (r < 0)
  2470. goto out_free_1;
  2471. }
  2472. r = register_cpu_notifier(&kvm_cpu_notifier);
  2473. if (r)
  2474. goto out_free_2;
  2475. register_reboot_notifier(&kvm_reboot_notifier);
  2476. /* A kmem cache lets us meet the alignment requirements of fx_save. */
  2477. if (!vcpu_align)
  2478. vcpu_align = __alignof__(struct kvm_vcpu);
  2479. kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
  2480. 0, NULL);
  2481. if (!kvm_vcpu_cache) {
  2482. r = -ENOMEM;
  2483. goto out_free_3;
  2484. }
  2485. r = kvm_async_pf_init();
  2486. if (r)
  2487. goto out_free;
  2488. kvm_chardev_ops.owner = module;
  2489. kvm_vm_fops.owner = module;
  2490. kvm_vcpu_fops.owner = module;
  2491. r = misc_register(&kvm_dev);
  2492. if (r) {
  2493. printk(KERN_ERR "kvm: misc device register failed\n");
  2494. goto out_unreg;
  2495. }
  2496. register_syscore_ops(&kvm_syscore_ops);
  2497. kvm_preempt_ops.sched_in = kvm_sched_in;
  2498. kvm_preempt_ops.sched_out = kvm_sched_out;
  2499. r = kvm_init_debug();
  2500. if (r) {
  2501. printk(KERN_ERR "kvm: create debugfs files failed\n");
  2502. goto out_undebugfs;
  2503. }
  2504. return 0;
  2505. out_undebugfs:
  2506. unregister_syscore_ops(&kvm_syscore_ops);
  2507. out_unreg:
  2508. kvm_async_pf_deinit();
  2509. out_free:
  2510. kmem_cache_destroy(kvm_vcpu_cache);
  2511. out_free_3:
  2512. unregister_reboot_notifier(&kvm_reboot_notifier);
  2513. unregister_cpu_notifier(&kvm_cpu_notifier);
  2514. out_free_2:
  2515. out_free_1:
  2516. kvm_arch_hardware_unsetup();
  2517. out_free_0a:
  2518. free_cpumask_var(cpus_hardware_enabled);
  2519. out_free_0:
  2520. kvm_arch_exit();
  2521. out_fail:
  2522. return r;
  2523. }
  2524. EXPORT_SYMBOL_GPL(kvm_init);
  2525. void kvm_exit(void)
  2526. {
  2527. kvm_exit_debug();
  2528. misc_deregister(&kvm_dev);
  2529. kmem_cache_destroy(kvm_vcpu_cache);
  2530. kvm_async_pf_deinit();
  2531. unregister_syscore_ops(&kvm_syscore_ops);
  2532. unregister_reboot_notifier(&kvm_reboot_notifier);
  2533. unregister_cpu_notifier(&kvm_cpu_notifier);
  2534. on_each_cpu(hardware_disable_nolock, NULL, 1);
  2535. kvm_arch_hardware_unsetup();
  2536. kvm_arch_exit();
  2537. free_cpumask_var(cpus_hardware_enabled);
  2538. }
  2539. EXPORT_SYMBOL_GPL(kvm_exit);