kvm_main.c 73 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. *
  9. * Authors:
  10. * Avi Kivity <avi@qumranet.com>
  11. * Yaniv Kamay <yaniv@qumranet.com>
  12. *
  13. * This work is licensed under the terms of the GNU GPL, version 2. See
  14. * the COPYING file in the top-level directory.
  15. *
  16. */
  17. #include "kvm.h"
  18. #include "x86_emulate.h"
  19. #include "segment_descriptor.h"
  20. #include "irq.h"
  21. #include <linux/kvm.h>
  22. #include <linux/module.h>
  23. #include <linux/errno.h>
  24. #include <linux/percpu.h>
  25. #include <linux/gfp.h>
  26. #include <linux/mm.h>
  27. #include <linux/miscdevice.h>
  28. #include <linux/vmalloc.h>
  29. #include <linux/reboot.h>
  30. #include <linux/debugfs.h>
  31. #include <linux/highmem.h>
  32. #include <linux/file.h>
  33. #include <linux/sysdev.h>
  34. #include <linux/cpu.h>
  35. #include <linux/sched.h>
  36. #include <linux/cpumask.h>
  37. #include <linux/smp.h>
  38. #include <linux/anon_inodes.h>
  39. #include <asm/processor.h>
  40. #include <asm/msr.h>
  41. #include <asm/io.h>
  42. #include <asm/uaccess.h>
  43. #include <asm/desc.h>
  44. MODULE_AUTHOR("Qumranet");
  45. MODULE_LICENSE("GPL");
  46. static DEFINE_SPINLOCK(kvm_lock);
  47. static LIST_HEAD(vm_list);
  48. static cpumask_t cpus_hardware_enabled;
  49. struct kvm_arch_ops *kvm_arch_ops;
  50. struct kmem_cache *kvm_vcpu_cache;
  51. EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  52. static __read_mostly struct preempt_ops kvm_preempt_ops;
  53. #define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x)
  54. static struct kvm_stats_debugfs_item {
  55. const char *name;
  56. int offset;
  57. struct dentry *dentry;
  58. } debugfs_entries[] = {
  59. { "pf_fixed", STAT_OFFSET(pf_fixed) },
  60. { "pf_guest", STAT_OFFSET(pf_guest) },
  61. { "tlb_flush", STAT_OFFSET(tlb_flush) },
  62. { "invlpg", STAT_OFFSET(invlpg) },
  63. { "exits", STAT_OFFSET(exits) },
  64. { "io_exits", STAT_OFFSET(io_exits) },
  65. { "mmio_exits", STAT_OFFSET(mmio_exits) },
  66. { "signal_exits", STAT_OFFSET(signal_exits) },
  67. { "irq_window", STAT_OFFSET(irq_window_exits) },
  68. { "halt_exits", STAT_OFFSET(halt_exits) },
  69. { "request_irq", STAT_OFFSET(request_irq_exits) },
  70. { "irq_exits", STAT_OFFSET(irq_exits) },
  71. { "light_exits", STAT_OFFSET(light_exits) },
  72. { "efer_reload", STAT_OFFSET(efer_reload) },
  73. { NULL }
  74. };
  75. static struct dentry *debugfs_dir;
  76. #define MAX_IO_MSRS 256
  77. #define CR0_RESERVED_BITS \
  78. (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
  79. | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
  80. | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
  81. #define CR4_RESERVED_BITS \
  82. (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
  83. | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
  84. | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \
  85. | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
  86. #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
  87. #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  88. #ifdef CONFIG_X86_64
  89. // LDT or TSS descriptor in the GDT. 16 bytes.
  90. struct segment_descriptor_64 {
  91. struct segment_descriptor s;
  92. u32 base_higher;
  93. u32 pad_zero;
  94. };
  95. #endif
  96. static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  97. unsigned long arg);
  98. unsigned long segment_base(u16 selector)
  99. {
  100. struct descriptor_table gdt;
  101. struct segment_descriptor *d;
  102. unsigned long table_base;
  103. typedef unsigned long ul;
  104. unsigned long v;
  105. if (selector == 0)
  106. return 0;
  107. asm ("sgdt %0" : "=m"(gdt));
  108. table_base = gdt.base;
  109. if (selector & 4) { /* from ldt */
  110. u16 ldt_selector;
  111. asm ("sldt %0" : "=g"(ldt_selector));
  112. table_base = segment_base(ldt_selector);
  113. }
  114. d = (struct segment_descriptor *)(table_base + (selector & ~7));
  115. v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
  116. #ifdef CONFIG_X86_64
  117. if (d->system == 0
  118. && (d->type == 2 || d->type == 9 || d->type == 11))
  119. v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
  120. #endif
  121. return v;
  122. }
  123. EXPORT_SYMBOL_GPL(segment_base);
  124. static inline int valid_vcpu(int n)
  125. {
  126. return likely(n >= 0 && n < KVM_MAX_VCPUS);
  127. }
  128. void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
  129. {
  130. if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
  131. return;
  132. vcpu->guest_fpu_loaded = 1;
  133. fx_save(&vcpu->host_fx_image);
  134. fx_restore(&vcpu->guest_fx_image);
  135. }
  136. EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
  137. void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
  138. {
  139. if (!vcpu->guest_fpu_loaded)
  140. return;
  141. vcpu->guest_fpu_loaded = 0;
  142. fx_save(&vcpu->guest_fx_image);
  143. fx_restore(&vcpu->host_fx_image);
  144. }
  145. EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
  146. /*
  147. * Switches to specified vcpu, until a matching vcpu_put()
  148. */
  149. static void vcpu_load(struct kvm_vcpu *vcpu)
  150. {
  151. int cpu;
  152. mutex_lock(&vcpu->mutex);
  153. cpu = get_cpu();
  154. preempt_notifier_register(&vcpu->preempt_notifier);
  155. kvm_arch_ops->vcpu_load(vcpu, cpu);
  156. put_cpu();
  157. }
  158. static void vcpu_put(struct kvm_vcpu *vcpu)
  159. {
  160. preempt_disable();
  161. kvm_arch_ops->vcpu_put(vcpu);
  162. preempt_notifier_unregister(&vcpu->preempt_notifier);
  163. preempt_enable();
  164. mutex_unlock(&vcpu->mutex);
  165. }
  166. static void ack_flush(void *_completed)
  167. {
  168. atomic_t *completed = _completed;
  169. atomic_inc(completed);
  170. }
  171. void kvm_flush_remote_tlbs(struct kvm *kvm)
  172. {
  173. int i, cpu, needed;
  174. cpumask_t cpus;
  175. struct kvm_vcpu *vcpu;
  176. atomic_t completed;
  177. atomic_set(&completed, 0);
  178. cpus_clear(cpus);
  179. needed = 0;
  180. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  181. vcpu = kvm->vcpus[i];
  182. if (!vcpu)
  183. continue;
  184. if (test_and_set_bit(KVM_TLB_FLUSH, &vcpu->requests))
  185. continue;
  186. cpu = vcpu->cpu;
  187. if (cpu != -1 && cpu != raw_smp_processor_id())
  188. if (!cpu_isset(cpu, cpus)) {
  189. cpu_set(cpu, cpus);
  190. ++needed;
  191. }
  192. }
  193. /*
  194. * We really want smp_call_function_mask() here. But that's not
  195. * available, so ipi all cpus in parallel and wait for them
  196. * to complete.
  197. */
  198. for (cpu = first_cpu(cpus); cpu != NR_CPUS; cpu = next_cpu(cpu, cpus))
  199. smp_call_function_single(cpu, ack_flush, &completed, 1, 0);
  200. while (atomic_read(&completed) != needed) {
  201. cpu_relax();
  202. barrier();
  203. }
  204. }
  205. int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
  206. {
  207. struct page *page;
  208. int r;
  209. mutex_init(&vcpu->mutex);
  210. vcpu->cpu = -1;
  211. vcpu->mmu.root_hpa = INVALID_PAGE;
  212. vcpu->kvm = kvm;
  213. vcpu->vcpu_id = id;
  214. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  215. if (!page) {
  216. r = -ENOMEM;
  217. goto fail;
  218. }
  219. vcpu->run = page_address(page);
  220. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  221. if (!page) {
  222. r = -ENOMEM;
  223. goto fail_free_run;
  224. }
  225. vcpu->pio_data = page_address(page);
  226. r = kvm_mmu_create(vcpu);
  227. if (r < 0)
  228. goto fail_free_pio_data;
  229. return 0;
  230. fail_free_pio_data:
  231. free_page((unsigned long)vcpu->pio_data);
  232. fail_free_run:
  233. free_page((unsigned long)vcpu->run);
  234. fail:
  235. return -ENOMEM;
  236. }
  237. EXPORT_SYMBOL_GPL(kvm_vcpu_init);
  238. void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
  239. {
  240. kvm_mmu_destroy(vcpu);
  241. kvm_free_apic(vcpu->apic);
  242. free_page((unsigned long)vcpu->pio_data);
  243. free_page((unsigned long)vcpu->run);
  244. }
  245. EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
  246. static struct kvm *kvm_create_vm(void)
  247. {
  248. struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
  249. if (!kvm)
  250. return ERR_PTR(-ENOMEM);
  251. kvm_io_bus_init(&kvm->pio_bus);
  252. mutex_init(&kvm->lock);
  253. INIT_LIST_HEAD(&kvm->active_mmu_pages);
  254. kvm_io_bus_init(&kvm->mmio_bus);
  255. spin_lock(&kvm_lock);
  256. list_add(&kvm->vm_list, &vm_list);
  257. spin_unlock(&kvm_lock);
  258. return kvm;
  259. }
  260. /*
  261. * Free any memory in @free but not in @dont.
  262. */
  263. static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
  264. struct kvm_memory_slot *dont)
  265. {
  266. int i;
  267. if (!dont || free->phys_mem != dont->phys_mem)
  268. if (free->phys_mem) {
  269. for (i = 0; i < free->npages; ++i)
  270. if (free->phys_mem[i])
  271. __free_page(free->phys_mem[i]);
  272. vfree(free->phys_mem);
  273. }
  274. if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
  275. vfree(free->dirty_bitmap);
  276. free->phys_mem = NULL;
  277. free->npages = 0;
  278. free->dirty_bitmap = NULL;
  279. }
  280. static void kvm_free_physmem(struct kvm *kvm)
  281. {
  282. int i;
  283. for (i = 0; i < kvm->nmemslots; ++i)
  284. kvm_free_physmem_slot(&kvm->memslots[i], NULL);
  285. }
  286. static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
  287. {
  288. int i;
  289. for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i)
  290. if (vcpu->pio.guest_pages[i]) {
  291. __free_page(vcpu->pio.guest_pages[i]);
  292. vcpu->pio.guest_pages[i] = NULL;
  293. }
  294. }
  295. static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
  296. {
  297. vcpu_load(vcpu);
  298. kvm_mmu_unload(vcpu);
  299. vcpu_put(vcpu);
  300. }
  301. static void kvm_free_vcpus(struct kvm *kvm)
  302. {
  303. unsigned int i;
  304. /*
  305. * Unpin any mmu pages first.
  306. */
  307. for (i = 0; i < KVM_MAX_VCPUS; ++i)
  308. if (kvm->vcpus[i])
  309. kvm_unload_vcpu_mmu(kvm->vcpus[i]);
  310. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  311. if (kvm->vcpus[i]) {
  312. kvm_arch_ops->vcpu_free(kvm->vcpus[i]);
  313. kvm->vcpus[i] = NULL;
  314. }
  315. }
  316. }
  317. static void kvm_destroy_vm(struct kvm *kvm)
  318. {
  319. spin_lock(&kvm_lock);
  320. list_del(&kvm->vm_list);
  321. spin_unlock(&kvm_lock);
  322. kvm_io_bus_destroy(&kvm->pio_bus);
  323. kvm_io_bus_destroy(&kvm->mmio_bus);
  324. kfree(kvm->vpic);
  325. kfree(kvm->vioapic);
  326. kvm_free_vcpus(kvm);
  327. kvm_free_physmem(kvm);
  328. kfree(kvm);
  329. }
  330. static int kvm_vm_release(struct inode *inode, struct file *filp)
  331. {
  332. struct kvm *kvm = filp->private_data;
  333. kvm_destroy_vm(kvm);
  334. return 0;
  335. }
  336. static void inject_gp(struct kvm_vcpu *vcpu)
  337. {
  338. kvm_arch_ops->inject_gp(vcpu, 0);
  339. }
  340. /*
  341. * Load the pae pdptrs. Return true is they are all valid.
  342. */
  343. static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
  344. {
  345. gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
  346. unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
  347. int i;
  348. u64 *pdpt;
  349. int ret;
  350. struct page *page;
  351. u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)];
  352. mutex_lock(&vcpu->kvm->lock);
  353. page = gfn_to_page(vcpu->kvm, pdpt_gfn);
  354. if (!page) {
  355. ret = 0;
  356. goto out;
  357. }
  358. pdpt = kmap_atomic(page, KM_USER0);
  359. memcpy(pdpte, pdpt+offset, sizeof(pdpte));
  360. kunmap_atomic(pdpt, KM_USER0);
  361. for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
  362. if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) {
  363. ret = 0;
  364. goto out;
  365. }
  366. }
  367. ret = 1;
  368. memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs));
  369. out:
  370. mutex_unlock(&vcpu->kvm->lock);
  371. return ret;
  372. }
  373. void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  374. {
  375. if (cr0 & CR0_RESERVED_BITS) {
  376. printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
  377. cr0, vcpu->cr0);
  378. inject_gp(vcpu);
  379. return;
  380. }
  381. if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
  382. printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
  383. inject_gp(vcpu);
  384. return;
  385. }
  386. if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
  387. printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
  388. "and a clear PE flag\n");
  389. inject_gp(vcpu);
  390. return;
  391. }
  392. if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
  393. #ifdef CONFIG_X86_64
  394. if ((vcpu->shadow_efer & EFER_LME)) {
  395. int cs_db, cs_l;
  396. if (!is_pae(vcpu)) {
  397. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  398. "in long mode while PAE is disabled\n");
  399. inject_gp(vcpu);
  400. return;
  401. }
  402. kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  403. if (cs_l) {
  404. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  405. "in long mode while CS.L == 1\n");
  406. inject_gp(vcpu);
  407. return;
  408. }
  409. } else
  410. #endif
  411. if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
  412. printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
  413. "reserved bits\n");
  414. inject_gp(vcpu);
  415. return;
  416. }
  417. }
  418. kvm_arch_ops->set_cr0(vcpu, cr0);
  419. vcpu->cr0 = cr0;
  420. mutex_lock(&vcpu->kvm->lock);
  421. kvm_mmu_reset_context(vcpu);
  422. mutex_unlock(&vcpu->kvm->lock);
  423. return;
  424. }
  425. EXPORT_SYMBOL_GPL(set_cr0);
  426. void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
  427. {
  428. set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
  429. }
  430. EXPORT_SYMBOL_GPL(lmsw);
  431. void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  432. {
  433. if (cr4 & CR4_RESERVED_BITS) {
  434. printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
  435. inject_gp(vcpu);
  436. return;
  437. }
  438. if (is_long_mode(vcpu)) {
  439. if (!(cr4 & X86_CR4_PAE)) {
  440. printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
  441. "in long mode\n");
  442. inject_gp(vcpu);
  443. return;
  444. }
  445. } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE)
  446. && !load_pdptrs(vcpu, vcpu->cr3)) {
  447. printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
  448. inject_gp(vcpu);
  449. return;
  450. }
  451. if (cr4 & X86_CR4_VMXE) {
  452. printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
  453. inject_gp(vcpu);
  454. return;
  455. }
  456. kvm_arch_ops->set_cr4(vcpu, cr4);
  457. mutex_lock(&vcpu->kvm->lock);
  458. kvm_mmu_reset_context(vcpu);
  459. mutex_unlock(&vcpu->kvm->lock);
  460. }
  461. EXPORT_SYMBOL_GPL(set_cr4);
  462. void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
  463. {
  464. if (is_long_mode(vcpu)) {
  465. if (cr3 & CR3_L_MODE_RESERVED_BITS) {
  466. printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
  467. inject_gp(vcpu);
  468. return;
  469. }
  470. } else {
  471. if (is_pae(vcpu)) {
  472. if (cr3 & CR3_PAE_RESERVED_BITS) {
  473. printk(KERN_DEBUG
  474. "set_cr3: #GP, reserved bits\n");
  475. inject_gp(vcpu);
  476. return;
  477. }
  478. if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
  479. printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
  480. "reserved bits\n");
  481. inject_gp(vcpu);
  482. return;
  483. }
  484. } else {
  485. if (cr3 & CR3_NONPAE_RESERVED_BITS) {
  486. printk(KERN_DEBUG
  487. "set_cr3: #GP, reserved bits\n");
  488. inject_gp(vcpu);
  489. return;
  490. }
  491. }
  492. }
  493. mutex_lock(&vcpu->kvm->lock);
  494. /*
  495. * Does the new cr3 value map to physical memory? (Note, we
  496. * catch an invalid cr3 even in real-mode, because it would
  497. * cause trouble later on when we turn on paging anyway.)
  498. *
  499. * A real CPU would silently accept an invalid cr3 and would
  500. * attempt to use it - with largely undefined (and often hard
  501. * to debug) behavior on the guest side.
  502. */
  503. if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
  504. inject_gp(vcpu);
  505. else {
  506. vcpu->cr3 = cr3;
  507. vcpu->mmu.new_cr3(vcpu);
  508. }
  509. mutex_unlock(&vcpu->kvm->lock);
  510. }
  511. EXPORT_SYMBOL_GPL(set_cr3);
  512. void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
  513. {
  514. if (cr8 & CR8_RESERVED_BITS) {
  515. printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
  516. inject_gp(vcpu);
  517. return;
  518. }
  519. if (irqchip_in_kernel(vcpu->kvm))
  520. kvm_lapic_set_tpr(vcpu, cr8);
  521. else
  522. vcpu->cr8 = cr8;
  523. }
  524. EXPORT_SYMBOL_GPL(set_cr8);
  525. unsigned long get_cr8(struct kvm_vcpu *vcpu)
  526. {
  527. if (irqchip_in_kernel(vcpu->kvm))
  528. return kvm_lapic_get_cr8(vcpu);
  529. else
  530. return vcpu->cr8;
  531. }
  532. EXPORT_SYMBOL_GPL(get_cr8);
  533. u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
  534. {
  535. if (irqchip_in_kernel(vcpu->kvm))
  536. return vcpu->apic_base;
  537. else
  538. return vcpu->apic_base;
  539. }
  540. EXPORT_SYMBOL_GPL(kvm_get_apic_base);
  541. void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
  542. {
  543. /* TODO: reserve bits check */
  544. if (irqchip_in_kernel(vcpu->kvm))
  545. kvm_lapic_set_base(vcpu, data);
  546. else
  547. vcpu->apic_base = data;
  548. }
  549. EXPORT_SYMBOL_GPL(kvm_set_apic_base);
  550. void fx_init(struct kvm_vcpu *vcpu)
  551. {
  552. unsigned after_mxcsr_mask;
  553. /* Initialize guest FPU by resetting ours and saving into guest's */
  554. preempt_disable();
  555. fx_save(&vcpu->host_fx_image);
  556. fpu_init();
  557. fx_save(&vcpu->guest_fx_image);
  558. fx_restore(&vcpu->host_fx_image);
  559. preempt_enable();
  560. after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
  561. vcpu->guest_fx_image.mxcsr = 0x1f80;
  562. memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask,
  563. 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
  564. }
  565. EXPORT_SYMBOL_GPL(fx_init);
  566. /*
  567. * Allocate some memory and give it an address in the guest physical address
  568. * space.
  569. *
  570. * Discontiguous memory is allowed, mostly for framebuffers.
  571. */
  572. static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  573. struct kvm_memory_region *mem)
  574. {
  575. int r;
  576. gfn_t base_gfn;
  577. unsigned long npages;
  578. unsigned long i;
  579. struct kvm_memory_slot *memslot;
  580. struct kvm_memory_slot old, new;
  581. int memory_config_version;
  582. r = -EINVAL;
  583. /* General sanity checks */
  584. if (mem->memory_size & (PAGE_SIZE - 1))
  585. goto out;
  586. if (mem->guest_phys_addr & (PAGE_SIZE - 1))
  587. goto out;
  588. if (mem->slot >= KVM_MEMORY_SLOTS)
  589. goto out;
  590. if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
  591. goto out;
  592. memslot = &kvm->memslots[mem->slot];
  593. base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
  594. npages = mem->memory_size >> PAGE_SHIFT;
  595. if (!npages)
  596. mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
  597. raced:
  598. mutex_lock(&kvm->lock);
  599. memory_config_version = kvm->memory_config_version;
  600. new = old = *memslot;
  601. new.base_gfn = base_gfn;
  602. new.npages = npages;
  603. new.flags = mem->flags;
  604. /* Disallow changing a memory slot's size. */
  605. r = -EINVAL;
  606. if (npages && old.npages && npages != old.npages)
  607. goto out_unlock;
  608. /* Check for overlaps */
  609. r = -EEXIST;
  610. for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
  611. struct kvm_memory_slot *s = &kvm->memslots[i];
  612. if (s == memslot)
  613. continue;
  614. if (!((base_gfn + npages <= s->base_gfn) ||
  615. (base_gfn >= s->base_gfn + s->npages)))
  616. goto out_unlock;
  617. }
  618. /*
  619. * Do memory allocations outside lock. memory_config_version will
  620. * detect any races.
  621. */
  622. mutex_unlock(&kvm->lock);
  623. /* Deallocate if slot is being removed */
  624. if (!npages)
  625. new.phys_mem = NULL;
  626. /* Free page dirty bitmap if unneeded */
  627. if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
  628. new.dirty_bitmap = NULL;
  629. r = -ENOMEM;
  630. /* Allocate if a slot is being created */
  631. if (npages && !new.phys_mem) {
  632. new.phys_mem = vmalloc(npages * sizeof(struct page *));
  633. if (!new.phys_mem)
  634. goto out_free;
  635. memset(new.phys_mem, 0, npages * sizeof(struct page *));
  636. for (i = 0; i < npages; ++i) {
  637. new.phys_mem[i] = alloc_page(GFP_HIGHUSER
  638. | __GFP_ZERO);
  639. if (!new.phys_mem[i])
  640. goto out_free;
  641. set_page_private(new.phys_mem[i],0);
  642. }
  643. }
  644. /* Allocate page dirty bitmap if needed */
  645. if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
  646. unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
  647. new.dirty_bitmap = vmalloc(dirty_bytes);
  648. if (!new.dirty_bitmap)
  649. goto out_free;
  650. memset(new.dirty_bitmap, 0, dirty_bytes);
  651. }
  652. mutex_lock(&kvm->lock);
  653. if (memory_config_version != kvm->memory_config_version) {
  654. mutex_unlock(&kvm->lock);
  655. kvm_free_physmem_slot(&new, &old);
  656. goto raced;
  657. }
  658. r = -EAGAIN;
  659. if (kvm->busy)
  660. goto out_unlock;
  661. if (mem->slot >= kvm->nmemslots)
  662. kvm->nmemslots = mem->slot + 1;
  663. *memslot = new;
  664. ++kvm->memory_config_version;
  665. kvm_mmu_slot_remove_write_access(kvm, mem->slot);
  666. kvm_flush_remote_tlbs(kvm);
  667. mutex_unlock(&kvm->lock);
  668. kvm_free_physmem_slot(&old, &new);
  669. return 0;
  670. out_unlock:
  671. mutex_unlock(&kvm->lock);
  672. out_free:
  673. kvm_free_physmem_slot(&new, &old);
  674. out:
  675. return r;
  676. }
  677. /*
  678. * Get (and clear) the dirty memory log for a memory slot.
  679. */
  680. static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
  681. struct kvm_dirty_log *log)
  682. {
  683. struct kvm_memory_slot *memslot;
  684. int r, i;
  685. int n;
  686. unsigned long any = 0;
  687. mutex_lock(&kvm->lock);
  688. /*
  689. * Prevent changes to guest memory configuration even while the lock
  690. * is not taken.
  691. */
  692. ++kvm->busy;
  693. mutex_unlock(&kvm->lock);
  694. r = -EINVAL;
  695. if (log->slot >= KVM_MEMORY_SLOTS)
  696. goto out;
  697. memslot = &kvm->memslots[log->slot];
  698. r = -ENOENT;
  699. if (!memslot->dirty_bitmap)
  700. goto out;
  701. n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
  702. for (i = 0; !any && i < n/sizeof(long); ++i)
  703. any = memslot->dirty_bitmap[i];
  704. r = -EFAULT;
  705. if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
  706. goto out;
  707. /* If nothing is dirty, don't bother messing with page tables. */
  708. if (any) {
  709. mutex_lock(&kvm->lock);
  710. kvm_mmu_slot_remove_write_access(kvm, log->slot);
  711. kvm_flush_remote_tlbs(kvm);
  712. memset(memslot->dirty_bitmap, 0, n);
  713. mutex_unlock(&kvm->lock);
  714. }
  715. r = 0;
  716. out:
  717. mutex_lock(&kvm->lock);
  718. --kvm->busy;
  719. mutex_unlock(&kvm->lock);
  720. return r;
  721. }
  722. /*
  723. * Set a new alias region. Aliases map a portion of physical memory into
  724. * another portion. This is useful for memory windows, for example the PC
  725. * VGA region.
  726. */
  727. static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
  728. struct kvm_memory_alias *alias)
  729. {
  730. int r, n;
  731. struct kvm_mem_alias *p;
  732. r = -EINVAL;
  733. /* General sanity checks */
  734. if (alias->memory_size & (PAGE_SIZE - 1))
  735. goto out;
  736. if (alias->guest_phys_addr & (PAGE_SIZE - 1))
  737. goto out;
  738. if (alias->slot >= KVM_ALIAS_SLOTS)
  739. goto out;
  740. if (alias->guest_phys_addr + alias->memory_size
  741. < alias->guest_phys_addr)
  742. goto out;
  743. if (alias->target_phys_addr + alias->memory_size
  744. < alias->target_phys_addr)
  745. goto out;
  746. mutex_lock(&kvm->lock);
  747. p = &kvm->aliases[alias->slot];
  748. p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
  749. p->npages = alias->memory_size >> PAGE_SHIFT;
  750. p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
  751. for (n = KVM_ALIAS_SLOTS; n > 0; --n)
  752. if (kvm->aliases[n - 1].npages)
  753. break;
  754. kvm->naliases = n;
  755. kvm_mmu_zap_all(kvm);
  756. mutex_unlock(&kvm->lock);
  757. return 0;
  758. out:
  759. return r;
  760. }
  761. static gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
  762. {
  763. int i;
  764. struct kvm_mem_alias *alias;
  765. for (i = 0; i < kvm->naliases; ++i) {
  766. alias = &kvm->aliases[i];
  767. if (gfn >= alias->base_gfn
  768. && gfn < alias->base_gfn + alias->npages)
  769. return alias->target_gfn + gfn - alias->base_gfn;
  770. }
  771. return gfn;
  772. }
  773. static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  774. {
  775. int i;
  776. for (i = 0; i < kvm->nmemslots; ++i) {
  777. struct kvm_memory_slot *memslot = &kvm->memslots[i];
  778. if (gfn >= memslot->base_gfn
  779. && gfn < memslot->base_gfn + memslot->npages)
  780. return memslot;
  781. }
  782. return NULL;
  783. }
  784. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  785. {
  786. gfn = unalias_gfn(kvm, gfn);
  787. return __gfn_to_memslot(kvm, gfn);
  788. }
  789. struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
  790. {
  791. struct kvm_memory_slot *slot;
  792. gfn = unalias_gfn(kvm, gfn);
  793. slot = __gfn_to_memslot(kvm, gfn);
  794. if (!slot)
  795. return NULL;
  796. return slot->phys_mem[gfn - slot->base_gfn];
  797. }
  798. EXPORT_SYMBOL_GPL(gfn_to_page);
  799. /* WARNING: Does not work on aliased pages. */
  800. void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
  801. {
  802. struct kvm_memory_slot *memslot;
  803. memslot = __gfn_to_memslot(kvm, gfn);
  804. if (memslot && memslot->dirty_bitmap) {
  805. unsigned long rel_gfn = gfn - memslot->base_gfn;
  806. /* avoid RMW */
  807. if (!test_bit(rel_gfn, memslot->dirty_bitmap))
  808. set_bit(rel_gfn, memslot->dirty_bitmap);
  809. }
  810. }
  811. int emulator_read_std(unsigned long addr,
  812. void *val,
  813. unsigned int bytes,
  814. struct kvm_vcpu *vcpu)
  815. {
  816. void *data = val;
  817. while (bytes) {
  818. gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
  819. unsigned offset = addr & (PAGE_SIZE-1);
  820. unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
  821. unsigned long pfn;
  822. struct page *page;
  823. void *page_virt;
  824. if (gpa == UNMAPPED_GVA)
  825. return X86EMUL_PROPAGATE_FAULT;
  826. pfn = gpa >> PAGE_SHIFT;
  827. page = gfn_to_page(vcpu->kvm, pfn);
  828. if (!page)
  829. return X86EMUL_UNHANDLEABLE;
  830. page_virt = kmap_atomic(page, KM_USER0);
  831. memcpy(data, page_virt + offset, tocopy);
  832. kunmap_atomic(page_virt, KM_USER0);
  833. bytes -= tocopy;
  834. data += tocopy;
  835. addr += tocopy;
  836. }
  837. return X86EMUL_CONTINUE;
  838. }
  839. EXPORT_SYMBOL_GPL(emulator_read_std);
  840. static int emulator_write_std(unsigned long addr,
  841. const void *val,
  842. unsigned int bytes,
  843. struct kvm_vcpu *vcpu)
  844. {
  845. pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes);
  846. return X86EMUL_UNHANDLEABLE;
  847. }
  848. /*
  849. * Only apic need an MMIO device hook, so shortcut now..
  850. */
  851. static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu,
  852. gpa_t addr)
  853. {
  854. struct kvm_io_device *dev;
  855. if (vcpu->apic) {
  856. dev = &vcpu->apic->dev;
  857. if (dev->in_range(dev, addr))
  858. return dev;
  859. }
  860. return NULL;
  861. }
  862. static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
  863. gpa_t addr)
  864. {
  865. struct kvm_io_device *dev;
  866. dev = vcpu_find_pervcpu_dev(vcpu, addr);
  867. if (dev == NULL)
  868. dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr);
  869. return dev;
  870. }
  871. static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
  872. gpa_t addr)
  873. {
  874. return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr);
  875. }
  876. static int emulator_read_emulated(unsigned long addr,
  877. void *val,
  878. unsigned int bytes,
  879. struct kvm_vcpu *vcpu)
  880. {
  881. struct kvm_io_device *mmio_dev;
  882. gpa_t gpa;
  883. if (vcpu->mmio_read_completed) {
  884. memcpy(val, vcpu->mmio_data, bytes);
  885. vcpu->mmio_read_completed = 0;
  886. return X86EMUL_CONTINUE;
  887. } else if (emulator_read_std(addr, val, bytes, vcpu)
  888. == X86EMUL_CONTINUE)
  889. return X86EMUL_CONTINUE;
  890. gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
  891. if (gpa == UNMAPPED_GVA)
  892. return X86EMUL_PROPAGATE_FAULT;
  893. /*
  894. * Is this MMIO handled locally?
  895. */
  896. mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
  897. if (mmio_dev) {
  898. kvm_iodevice_read(mmio_dev, gpa, bytes, val);
  899. return X86EMUL_CONTINUE;
  900. }
  901. vcpu->mmio_needed = 1;
  902. vcpu->mmio_phys_addr = gpa;
  903. vcpu->mmio_size = bytes;
  904. vcpu->mmio_is_write = 0;
  905. return X86EMUL_UNHANDLEABLE;
  906. }
  907. static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
  908. const void *val, int bytes)
  909. {
  910. struct page *page;
  911. void *virt;
  912. if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
  913. return 0;
  914. page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  915. if (!page)
  916. return 0;
  917. mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
  918. virt = kmap_atomic(page, KM_USER0);
  919. kvm_mmu_pte_write(vcpu, gpa, val, bytes);
  920. memcpy(virt + offset_in_page(gpa), val, bytes);
  921. kunmap_atomic(virt, KM_USER0);
  922. return 1;
  923. }
  924. static int emulator_write_emulated_onepage(unsigned long addr,
  925. const void *val,
  926. unsigned int bytes,
  927. struct kvm_vcpu *vcpu)
  928. {
  929. struct kvm_io_device *mmio_dev;
  930. gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
  931. if (gpa == UNMAPPED_GVA) {
  932. kvm_arch_ops->inject_page_fault(vcpu, addr, 2);
  933. return X86EMUL_PROPAGATE_FAULT;
  934. }
  935. if (emulator_write_phys(vcpu, gpa, val, bytes))
  936. return X86EMUL_CONTINUE;
  937. /*
  938. * Is this MMIO handled locally?
  939. */
  940. mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
  941. if (mmio_dev) {
  942. kvm_iodevice_write(mmio_dev, gpa, bytes, val);
  943. return X86EMUL_CONTINUE;
  944. }
  945. vcpu->mmio_needed = 1;
  946. vcpu->mmio_phys_addr = gpa;
  947. vcpu->mmio_size = bytes;
  948. vcpu->mmio_is_write = 1;
  949. memcpy(vcpu->mmio_data, val, bytes);
  950. return X86EMUL_CONTINUE;
  951. }
  952. int emulator_write_emulated(unsigned long addr,
  953. const void *val,
  954. unsigned int bytes,
  955. struct kvm_vcpu *vcpu)
  956. {
  957. /* Crossing a page boundary? */
  958. if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
  959. int rc, now;
  960. now = -addr & ~PAGE_MASK;
  961. rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
  962. if (rc != X86EMUL_CONTINUE)
  963. return rc;
  964. addr += now;
  965. val += now;
  966. bytes -= now;
  967. }
  968. return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
  969. }
  970. EXPORT_SYMBOL_GPL(emulator_write_emulated);
  971. static int emulator_cmpxchg_emulated(unsigned long addr,
  972. const void *old,
  973. const void *new,
  974. unsigned int bytes,
  975. struct kvm_vcpu *vcpu)
  976. {
  977. static int reported;
  978. if (!reported) {
  979. reported = 1;
  980. printk(KERN_WARNING "kvm: emulating exchange as write\n");
  981. }
  982. return emulator_write_emulated(addr, new, bytes, vcpu);
  983. }
  984. static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
  985. {
  986. return kvm_arch_ops->get_segment_base(vcpu, seg);
  987. }
  988. int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
  989. {
  990. return X86EMUL_CONTINUE;
  991. }
  992. int emulate_clts(struct kvm_vcpu *vcpu)
  993. {
  994. unsigned long cr0;
  995. cr0 = vcpu->cr0 & ~X86_CR0_TS;
  996. kvm_arch_ops->set_cr0(vcpu, cr0);
  997. return X86EMUL_CONTINUE;
  998. }
  999. int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
  1000. {
  1001. struct kvm_vcpu *vcpu = ctxt->vcpu;
  1002. switch (dr) {
  1003. case 0 ... 3:
  1004. *dest = kvm_arch_ops->get_dr(vcpu, dr);
  1005. return X86EMUL_CONTINUE;
  1006. default:
  1007. pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr);
  1008. return X86EMUL_UNHANDLEABLE;
  1009. }
  1010. }
  1011. int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
  1012. {
  1013. unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
  1014. int exception;
  1015. kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
  1016. if (exception) {
  1017. /* FIXME: better handling */
  1018. return X86EMUL_UNHANDLEABLE;
  1019. }
  1020. return X86EMUL_CONTINUE;
  1021. }
  1022. static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
  1023. {
  1024. static int reported;
  1025. u8 opcodes[4];
  1026. unsigned long rip = ctxt->vcpu->rip;
  1027. unsigned long rip_linear;
  1028. rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
  1029. if (reported)
  1030. return;
  1031. emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt->vcpu);
  1032. printk(KERN_ERR "emulation failed but !mmio_needed?"
  1033. " rip %lx %02x %02x %02x %02x\n",
  1034. rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
  1035. reported = 1;
  1036. }
  1037. struct x86_emulate_ops emulate_ops = {
  1038. .read_std = emulator_read_std,
  1039. .write_std = emulator_write_std,
  1040. .read_emulated = emulator_read_emulated,
  1041. .write_emulated = emulator_write_emulated,
  1042. .cmpxchg_emulated = emulator_cmpxchg_emulated,
  1043. };
  1044. int emulate_instruction(struct kvm_vcpu *vcpu,
  1045. struct kvm_run *run,
  1046. unsigned long cr2,
  1047. u16 error_code)
  1048. {
  1049. struct x86_emulate_ctxt emulate_ctxt;
  1050. int r;
  1051. int cs_db, cs_l;
  1052. vcpu->mmio_fault_cr2 = cr2;
  1053. kvm_arch_ops->cache_regs(vcpu);
  1054. kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  1055. emulate_ctxt.vcpu = vcpu;
  1056. emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
  1057. emulate_ctxt.cr2 = cr2;
  1058. emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
  1059. ? X86EMUL_MODE_REAL : cs_l
  1060. ? X86EMUL_MODE_PROT64 : cs_db
  1061. ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
  1062. if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
  1063. emulate_ctxt.cs_base = 0;
  1064. emulate_ctxt.ds_base = 0;
  1065. emulate_ctxt.es_base = 0;
  1066. emulate_ctxt.ss_base = 0;
  1067. } else {
  1068. emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
  1069. emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
  1070. emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
  1071. emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
  1072. }
  1073. emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
  1074. emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
  1075. vcpu->mmio_is_write = 0;
  1076. vcpu->pio.string = 0;
  1077. r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
  1078. if (vcpu->pio.string)
  1079. return EMULATE_DO_MMIO;
  1080. if ((r || vcpu->mmio_is_write) && run) {
  1081. run->exit_reason = KVM_EXIT_MMIO;
  1082. run->mmio.phys_addr = vcpu->mmio_phys_addr;
  1083. memcpy(run->mmio.data, vcpu->mmio_data, 8);
  1084. run->mmio.len = vcpu->mmio_size;
  1085. run->mmio.is_write = vcpu->mmio_is_write;
  1086. }
  1087. if (r) {
  1088. if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
  1089. return EMULATE_DONE;
  1090. if (!vcpu->mmio_needed) {
  1091. report_emulation_failure(&emulate_ctxt);
  1092. return EMULATE_FAIL;
  1093. }
  1094. return EMULATE_DO_MMIO;
  1095. }
  1096. kvm_arch_ops->decache_regs(vcpu);
  1097. kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
  1098. if (vcpu->mmio_is_write) {
  1099. vcpu->mmio_needed = 0;
  1100. return EMULATE_DO_MMIO;
  1101. }
  1102. return EMULATE_DONE;
  1103. }
  1104. EXPORT_SYMBOL_GPL(emulate_instruction);
  1105. int kvm_emulate_halt(struct kvm_vcpu *vcpu)
  1106. {
  1107. if (vcpu->irq_summary ||
  1108. (irqchip_in_kernel(vcpu->kvm) && kvm_cpu_has_interrupt(vcpu)))
  1109. return 1;
  1110. vcpu->run->exit_reason = KVM_EXIT_HLT;
  1111. ++vcpu->stat.halt_exits;
  1112. return 0;
  1113. }
  1114. EXPORT_SYMBOL_GPL(kvm_emulate_halt);
  1115. int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
  1116. {
  1117. unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
  1118. kvm_arch_ops->cache_regs(vcpu);
  1119. ret = -KVM_EINVAL;
  1120. #ifdef CONFIG_X86_64
  1121. if (is_long_mode(vcpu)) {
  1122. nr = vcpu->regs[VCPU_REGS_RAX];
  1123. a0 = vcpu->regs[VCPU_REGS_RDI];
  1124. a1 = vcpu->regs[VCPU_REGS_RSI];
  1125. a2 = vcpu->regs[VCPU_REGS_RDX];
  1126. a3 = vcpu->regs[VCPU_REGS_RCX];
  1127. a4 = vcpu->regs[VCPU_REGS_R8];
  1128. a5 = vcpu->regs[VCPU_REGS_R9];
  1129. } else
  1130. #endif
  1131. {
  1132. nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
  1133. a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
  1134. a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
  1135. a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
  1136. a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
  1137. a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
  1138. a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
  1139. }
  1140. switch (nr) {
  1141. default:
  1142. run->hypercall.nr = nr;
  1143. run->hypercall.args[0] = a0;
  1144. run->hypercall.args[1] = a1;
  1145. run->hypercall.args[2] = a2;
  1146. run->hypercall.args[3] = a3;
  1147. run->hypercall.args[4] = a4;
  1148. run->hypercall.args[5] = a5;
  1149. run->hypercall.ret = ret;
  1150. run->hypercall.longmode = is_long_mode(vcpu);
  1151. kvm_arch_ops->decache_regs(vcpu);
  1152. return 0;
  1153. }
  1154. vcpu->regs[VCPU_REGS_RAX] = ret;
  1155. kvm_arch_ops->decache_regs(vcpu);
  1156. return 1;
  1157. }
  1158. EXPORT_SYMBOL_GPL(kvm_hypercall);
  1159. static u64 mk_cr_64(u64 curr_cr, u32 new_val)
  1160. {
  1161. return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
  1162. }
  1163. void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  1164. {
  1165. struct descriptor_table dt = { limit, base };
  1166. kvm_arch_ops->set_gdt(vcpu, &dt);
  1167. }
  1168. void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  1169. {
  1170. struct descriptor_table dt = { limit, base };
  1171. kvm_arch_ops->set_idt(vcpu, &dt);
  1172. }
  1173. void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
  1174. unsigned long *rflags)
  1175. {
  1176. lmsw(vcpu, msw);
  1177. *rflags = kvm_arch_ops->get_rflags(vcpu);
  1178. }
  1179. unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
  1180. {
  1181. kvm_arch_ops->decache_cr4_guest_bits(vcpu);
  1182. switch (cr) {
  1183. case 0:
  1184. return vcpu->cr0;
  1185. case 2:
  1186. return vcpu->cr2;
  1187. case 3:
  1188. return vcpu->cr3;
  1189. case 4:
  1190. return vcpu->cr4;
  1191. default:
  1192. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
  1193. return 0;
  1194. }
  1195. }
  1196. void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
  1197. unsigned long *rflags)
  1198. {
  1199. switch (cr) {
  1200. case 0:
  1201. set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
  1202. *rflags = kvm_arch_ops->get_rflags(vcpu);
  1203. break;
  1204. case 2:
  1205. vcpu->cr2 = val;
  1206. break;
  1207. case 3:
  1208. set_cr3(vcpu, val);
  1209. break;
  1210. case 4:
  1211. set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
  1212. break;
  1213. default:
  1214. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
  1215. }
  1216. }
  1217. /*
  1218. * Register the para guest with the host:
  1219. */
  1220. static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
  1221. {
  1222. struct kvm_vcpu_para_state *para_state;
  1223. hpa_t para_state_hpa, hypercall_hpa;
  1224. struct page *para_state_page;
  1225. unsigned char *hypercall;
  1226. gpa_t hypercall_gpa;
  1227. printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
  1228. printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
  1229. /*
  1230. * Needs to be page aligned:
  1231. */
  1232. if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
  1233. goto err_gp;
  1234. para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
  1235. printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
  1236. if (is_error_hpa(para_state_hpa))
  1237. goto err_gp;
  1238. mark_page_dirty(vcpu->kvm, para_state_gpa >> PAGE_SHIFT);
  1239. para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
  1240. para_state = kmap(para_state_page);
  1241. printk(KERN_DEBUG ".... guest version: %d\n", para_state->guest_version);
  1242. printk(KERN_DEBUG ".... size: %d\n", para_state->size);
  1243. para_state->host_version = KVM_PARA_API_VERSION;
  1244. /*
  1245. * We cannot support guests that try to register themselves
  1246. * with a newer API version than the host supports:
  1247. */
  1248. if (para_state->guest_version > KVM_PARA_API_VERSION) {
  1249. para_state->ret = -KVM_EINVAL;
  1250. goto err_kunmap_skip;
  1251. }
  1252. hypercall_gpa = para_state->hypercall_gpa;
  1253. hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
  1254. printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
  1255. if (is_error_hpa(hypercall_hpa)) {
  1256. para_state->ret = -KVM_EINVAL;
  1257. goto err_kunmap_skip;
  1258. }
  1259. printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
  1260. vcpu->para_state_page = para_state_page;
  1261. vcpu->para_state_gpa = para_state_gpa;
  1262. vcpu->hypercall_gpa = hypercall_gpa;
  1263. mark_page_dirty(vcpu->kvm, hypercall_gpa >> PAGE_SHIFT);
  1264. hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
  1265. KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
  1266. kvm_arch_ops->patch_hypercall(vcpu, hypercall);
  1267. kunmap_atomic(hypercall, KM_USER1);
  1268. para_state->ret = 0;
  1269. err_kunmap_skip:
  1270. kunmap(para_state_page);
  1271. return 0;
  1272. err_gp:
  1273. return 1;
  1274. }
  1275. int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1276. {
  1277. u64 data;
  1278. switch (msr) {
  1279. case 0xc0010010: /* SYSCFG */
  1280. case 0xc0010015: /* HWCR */
  1281. case MSR_IA32_PLATFORM_ID:
  1282. case MSR_IA32_P5_MC_ADDR:
  1283. case MSR_IA32_P5_MC_TYPE:
  1284. case MSR_IA32_MC0_CTL:
  1285. case MSR_IA32_MCG_STATUS:
  1286. case MSR_IA32_MCG_CAP:
  1287. case MSR_IA32_MC0_MISC:
  1288. case MSR_IA32_MC0_MISC+4:
  1289. case MSR_IA32_MC0_MISC+8:
  1290. case MSR_IA32_MC0_MISC+12:
  1291. case MSR_IA32_MC0_MISC+16:
  1292. case MSR_IA32_UCODE_REV:
  1293. case MSR_IA32_PERF_STATUS:
  1294. case MSR_IA32_EBL_CR_POWERON:
  1295. /* MTRR registers */
  1296. case 0xfe:
  1297. case 0x200 ... 0x2ff:
  1298. data = 0;
  1299. break;
  1300. case 0xcd: /* fsb frequency */
  1301. data = 3;
  1302. break;
  1303. case MSR_IA32_APICBASE:
  1304. data = kvm_get_apic_base(vcpu);
  1305. break;
  1306. case MSR_IA32_MISC_ENABLE:
  1307. data = vcpu->ia32_misc_enable_msr;
  1308. break;
  1309. #ifdef CONFIG_X86_64
  1310. case MSR_EFER:
  1311. data = vcpu->shadow_efer;
  1312. break;
  1313. #endif
  1314. default:
  1315. pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
  1316. return 1;
  1317. }
  1318. *pdata = data;
  1319. return 0;
  1320. }
  1321. EXPORT_SYMBOL_GPL(kvm_get_msr_common);
  1322. /*
  1323. * Reads an msr value (of 'msr_index') into 'pdata'.
  1324. * Returns 0 on success, non-0 otherwise.
  1325. * Assumes vcpu_load() was already called.
  1326. */
  1327. int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
  1328. {
  1329. return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
  1330. }
  1331. #ifdef CONFIG_X86_64
  1332. static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
  1333. {
  1334. if (efer & EFER_RESERVED_BITS) {
  1335. printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
  1336. efer);
  1337. inject_gp(vcpu);
  1338. return;
  1339. }
  1340. if (is_paging(vcpu)
  1341. && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
  1342. printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
  1343. inject_gp(vcpu);
  1344. return;
  1345. }
  1346. kvm_arch_ops->set_efer(vcpu, efer);
  1347. efer &= ~EFER_LMA;
  1348. efer |= vcpu->shadow_efer & EFER_LMA;
  1349. vcpu->shadow_efer = efer;
  1350. }
  1351. #endif
  1352. int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1353. {
  1354. switch (msr) {
  1355. #ifdef CONFIG_X86_64
  1356. case MSR_EFER:
  1357. set_efer(vcpu, data);
  1358. break;
  1359. #endif
  1360. case MSR_IA32_MC0_STATUS:
  1361. pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
  1362. __FUNCTION__, data);
  1363. break;
  1364. case MSR_IA32_MCG_STATUS:
  1365. pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
  1366. __FUNCTION__, data);
  1367. break;
  1368. case MSR_IA32_UCODE_REV:
  1369. case MSR_IA32_UCODE_WRITE:
  1370. case 0x200 ... 0x2ff: /* MTRRs */
  1371. break;
  1372. case MSR_IA32_APICBASE:
  1373. kvm_set_apic_base(vcpu, data);
  1374. break;
  1375. case MSR_IA32_MISC_ENABLE:
  1376. vcpu->ia32_misc_enable_msr = data;
  1377. break;
  1378. /*
  1379. * This is the 'probe whether the host is KVM' logic:
  1380. */
  1381. case MSR_KVM_API_MAGIC:
  1382. return vcpu_register_para(vcpu, data);
  1383. default:
  1384. pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr);
  1385. return 1;
  1386. }
  1387. return 0;
  1388. }
  1389. EXPORT_SYMBOL_GPL(kvm_set_msr_common);
  1390. /*
  1391. * Writes msr value into into the appropriate "register".
  1392. * Returns 0 on success, non-0 otherwise.
  1393. * Assumes vcpu_load() was already called.
  1394. */
  1395. int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
  1396. {
  1397. return kvm_arch_ops->set_msr(vcpu, msr_index, data);
  1398. }
  1399. void kvm_resched(struct kvm_vcpu *vcpu)
  1400. {
  1401. if (!need_resched())
  1402. return;
  1403. cond_resched();
  1404. }
  1405. EXPORT_SYMBOL_GPL(kvm_resched);
  1406. void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
  1407. {
  1408. int i;
  1409. u32 function;
  1410. struct kvm_cpuid_entry *e, *best;
  1411. kvm_arch_ops->cache_regs(vcpu);
  1412. function = vcpu->regs[VCPU_REGS_RAX];
  1413. vcpu->regs[VCPU_REGS_RAX] = 0;
  1414. vcpu->regs[VCPU_REGS_RBX] = 0;
  1415. vcpu->regs[VCPU_REGS_RCX] = 0;
  1416. vcpu->regs[VCPU_REGS_RDX] = 0;
  1417. best = NULL;
  1418. for (i = 0; i < vcpu->cpuid_nent; ++i) {
  1419. e = &vcpu->cpuid_entries[i];
  1420. if (e->function == function) {
  1421. best = e;
  1422. break;
  1423. }
  1424. /*
  1425. * Both basic or both extended?
  1426. */
  1427. if (((e->function ^ function) & 0x80000000) == 0)
  1428. if (!best || e->function > best->function)
  1429. best = e;
  1430. }
  1431. if (best) {
  1432. vcpu->regs[VCPU_REGS_RAX] = best->eax;
  1433. vcpu->regs[VCPU_REGS_RBX] = best->ebx;
  1434. vcpu->regs[VCPU_REGS_RCX] = best->ecx;
  1435. vcpu->regs[VCPU_REGS_RDX] = best->edx;
  1436. }
  1437. kvm_arch_ops->decache_regs(vcpu);
  1438. kvm_arch_ops->skip_emulated_instruction(vcpu);
  1439. }
  1440. EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
  1441. static int pio_copy_data(struct kvm_vcpu *vcpu)
  1442. {
  1443. void *p = vcpu->pio_data;
  1444. void *q;
  1445. unsigned bytes;
  1446. int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1;
  1447. q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
  1448. PAGE_KERNEL);
  1449. if (!q) {
  1450. free_pio_guest_pages(vcpu);
  1451. return -ENOMEM;
  1452. }
  1453. q += vcpu->pio.guest_page_offset;
  1454. bytes = vcpu->pio.size * vcpu->pio.cur_count;
  1455. if (vcpu->pio.in)
  1456. memcpy(q, p, bytes);
  1457. else
  1458. memcpy(p, q, bytes);
  1459. q -= vcpu->pio.guest_page_offset;
  1460. vunmap(q);
  1461. free_pio_guest_pages(vcpu);
  1462. return 0;
  1463. }
  1464. static int complete_pio(struct kvm_vcpu *vcpu)
  1465. {
  1466. struct kvm_pio_request *io = &vcpu->pio;
  1467. long delta;
  1468. int r;
  1469. kvm_arch_ops->cache_regs(vcpu);
  1470. if (!io->string) {
  1471. if (io->in)
  1472. memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data,
  1473. io->size);
  1474. } else {
  1475. if (io->in) {
  1476. r = pio_copy_data(vcpu);
  1477. if (r) {
  1478. kvm_arch_ops->cache_regs(vcpu);
  1479. return r;
  1480. }
  1481. }
  1482. delta = 1;
  1483. if (io->rep) {
  1484. delta *= io->cur_count;
  1485. /*
  1486. * The size of the register should really depend on
  1487. * current address size.
  1488. */
  1489. vcpu->regs[VCPU_REGS_RCX] -= delta;
  1490. }
  1491. if (io->down)
  1492. delta = -delta;
  1493. delta *= io->size;
  1494. if (io->in)
  1495. vcpu->regs[VCPU_REGS_RDI] += delta;
  1496. else
  1497. vcpu->regs[VCPU_REGS_RSI] += delta;
  1498. }
  1499. kvm_arch_ops->decache_regs(vcpu);
  1500. io->count -= io->cur_count;
  1501. io->cur_count = 0;
  1502. if (!io->count)
  1503. kvm_arch_ops->skip_emulated_instruction(vcpu);
  1504. return 0;
  1505. }
  1506. static void kernel_pio(struct kvm_io_device *pio_dev,
  1507. struct kvm_vcpu *vcpu,
  1508. void *pd)
  1509. {
  1510. /* TODO: String I/O for in kernel device */
  1511. if (vcpu->pio.in)
  1512. kvm_iodevice_read(pio_dev, vcpu->pio.port,
  1513. vcpu->pio.size,
  1514. pd);
  1515. else
  1516. kvm_iodevice_write(pio_dev, vcpu->pio.port,
  1517. vcpu->pio.size,
  1518. pd);
  1519. }
  1520. static void pio_string_write(struct kvm_io_device *pio_dev,
  1521. struct kvm_vcpu *vcpu)
  1522. {
  1523. struct kvm_pio_request *io = &vcpu->pio;
  1524. void *pd = vcpu->pio_data;
  1525. int i;
  1526. for (i = 0; i < io->cur_count; i++) {
  1527. kvm_iodevice_write(pio_dev, io->port,
  1528. io->size,
  1529. pd);
  1530. pd += io->size;
  1531. }
  1532. }
  1533. int kvm_emulate_pio (struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
  1534. int size, unsigned port)
  1535. {
  1536. struct kvm_io_device *pio_dev;
  1537. vcpu->run->exit_reason = KVM_EXIT_IO;
  1538. vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  1539. vcpu->run->io.size = vcpu->pio.size = size;
  1540. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  1541. vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1;
  1542. vcpu->run->io.port = vcpu->pio.port = port;
  1543. vcpu->pio.in = in;
  1544. vcpu->pio.string = 0;
  1545. vcpu->pio.down = 0;
  1546. vcpu->pio.guest_page_offset = 0;
  1547. vcpu->pio.rep = 0;
  1548. kvm_arch_ops->cache_regs(vcpu);
  1549. memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4);
  1550. kvm_arch_ops->decache_regs(vcpu);
  1551. pio_dev = vcpu_find_pio_dev(vcpu, port);
  1552. if (pio_dev) {
  1553. kernel_pio(pio_dev, vcpu, vcpu->pio_data);
  1554. complete_pio(vcpu);
  1555. return 1;
  1556. }
  1557. return 0;
  1558. }
  1559. EXPORT_SYMBOL_GPL(kvm_emulate_pio);
  1560. int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
  1561. int size, unsigned long count, int down,
  1562. gva_t address, int rep, unsigned port)
  1563. {
  1564. unsigned now, in_page;
  1565. int i, ret = 0;
  1566. int nr_pages = 1;
  1567. struct page *page;
  1568. struct kvm_io_device *pio_dev;
  1569. vcpu->run->exit_reason = KVM_EXIT_IO;
  1570. vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  1571. vcpu->run->io.size = vcpu->pio.size = size;
  1572. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  1573. vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count;
  1574. vcpu->run->io.port = vcpu->pio.port = port;
  1575. vcpu->pio.in = in;
  1576. vcpu->pio.string = 1;
  1577. vcpu->pio.down = down;
  1578. vcpu->pio.guest_page_offset = offset_in_page(address);
  1579. vcpu->pio.rep = rep;
  1580. if (!count) {
  1581. kvm_arch_ops->skip_emulated_instruction(vcpu);
  1582. return 1;
  1583. }
  1584. if (!down)
  1585. in_page = PAGE_SIZE - offset_in_page(address);
  1586. else
  1587. in_page = offset_in_page(address) + size;
  1588. now = min(count, (unsigned long)in_page / size);
  1589. if (!now) {
  1590. /*
  1591. * String I/O straddles page boundary. Pin two guest pages
  1592. * so that we satisfy atomicity constraints. Do just one
  1593. * transaction to avoid complexity.
  1594. */
  1595. nr_pages = 2;
  1596. now = 1;
  1597. }
  1598. if (down) {
  1599. /*
  1600. * String I/O in reverse. Yuck. Kill the guest, fix later.
  1601. */
  1602. pr_unimpl(vcpu, "guest string pio down\n");
  1603. inject_gp(vcpu);
  1604. return 1;
  1605. }
  1606. vcpu->run->io.count = now;
  1607. vcpu->pio.cur_count = now;
  1608. for (i = 0; i < nr_pages; ++i) {
  1609. mutex_lock(&vcpu->kvm->lock);
  1610. page = gva_to_page(vcpu, address + i * PAGE_SIZE);
  1611. if (page)
  1612. get_page(page);
  1613. vcpu->pio.guest_pages[i] = page;
  1614. mutex_unlock(&vcpu->kvm->lock);
  1615. if (!page) {
  1616. inject_gp(vcpu);
  1617. free_pio_guest_pages(vcpu);
  1618. return 1;
  1619. }
  1620. }
  1621. pio_dev = vcpu_find_pio_dev(vcpu, port);
  1622. if (!vcpu->pio.in) {
  1623. /* string PIO write */
  1624. ret = pio_copy_data(vcpu);
  1625. if (ret >= 0 && pio_dev) {
  1626. pio_string_write(pio_dev, vcpu);
  1627. complete_pio(vcpu);
  1628. if (vcpu->pio.count == 0)
  1629. ret = 1;
  1630. }
  1631. } else if (pio_dev)
  1632. pr_unimpl(vcpu, "no string pio read support yet, "
  1633. "port %x size %d count %ld\n",
  1634. port, size, count);
  1635. return ret;
  1636. }
  1637. EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
  1638. static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1639. {
  1640. int r;
  1641. sigset_t sigsaved;
  1642. vcpu_load(vcpu);
  1643. if (vcpu->sigset_active)
  1644. sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
  1645. /* re-sync apic's tpr */
  1646. set_cr8(vcpu, kvm_run->cr8);
  1647. if (vcpu->pio.cur_count) {
  1648. r = complete_pio(vcpu);
  1649. if (r)
  1650. goto out;
  1651. }
  1652. if (vcpu->mmio_needed) {
  1653. memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
  1654. vcpu->mmio_read_completed = 1;
  1655. vcpu->mmio_needed = 0;
  1656. r = emulate_instruction(vcpu, kvm_run,
  1657. vcpu->mmio_fault_cr2, 0);
  1658. if (r == EMULATE_DO_MMIO) {
  1659. /*
  1660. * Read-modify-write. Back to userspace.
  1661. */
  1662. r = 0;
  1663. goto out;
  1664. }
  1665. }
  1666. if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
  1667. kvm_arch_ops->cache_regs(vcpu);
  1668. vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
  1669. kvm_arch_ops->decache_regs(vcpu);
  1670. }
  1671. r = kvm_arch_ops->run(vcpu, kvm_run);
  1672. out:
  1673. if (vcpu->sigset_active)
  1674. sigprocmask(SIG_SETMASK, &sigsaved, NULL);
  1675. vcpu_put(vcpu);
  1676. return r;
  1677. }
  1678. static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
  1679. struct kvm_regs *regs)
  1680. {
  1681. vcpu_load(vcpu);
  1682. kvm_arch_ops->cache_regs(vcpu);
  1683. regs->rax = vcpu->regs[VCPU_REGS_RAX];
  1684. regs->rbx = vcpu->regs[VCPU_REGS_RBX];
  1685. regs->rcx = vcpu->regs[VCPU_REGS_RCX];
  1686. regs->rdx = vcpu->regs[VCPU_REGS_RDX];
  1687. regs->rsi = vcpu->regs[VCPU_REGS_RSI];
  1688. regs->rdi = vcpu->regs[VCPU_REGS_RDI];
  1689. regs->rsp = vcpu->regs[VCPU_REGS_RSP];
  1690. regs->rbp = vcpu->regs[VCPU_REGS_RBP];
  1691. #ifdef CONFIG_X86_64
  1692. regs->r8 = vcpu->regs[VCPU_REGS_R8];
  1693. regs->r9 = vcpu->regs[VCPU_REGS_R9];
  1694. regs->r10 = vcpu->regs[VCPU_REGS_R10];
  1695. regs->r11 = vcpu->regs[VCPU_REGS_R11];
  1696. regs->r12 = vcpu->regs[VCPU_REGS_R12];
  1697. regs->r13 = vcpu->regs[VCPU_REGS_R13];
  1698. regs->r14 = vcpu->regs[VCPU_REGS_R14];
  1699. regs->r15 = vcpu->regs[VCPU_REGS_R15];
  1700. #endif
  1701. regs->rip = vcpu->rip;
  1702. regs->rflags = kvm_arch_ops->get_rflags(vcpu);
  1703. /*
  1704. * Don't leak debug flags in case they were set for guest debugging
  1705. */
  1706. if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
  1707. regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
  1708. vcpu_put(vcpu);
  1709. return 0;
  1710. }
  1711. static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
  1712. struct kvm_regs *regs)
  1713. {
  1714. vcpu_load(vcpu);
  1715. vcpu->regs[VCPU_REGS_RAX] = regs->rax;
  1716. vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
  1717. vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
  1718. vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
  1719. vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
  1720. vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
  1721. vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
  1722. vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
  1723. #ifdef CONFIG_X86_64
  1724. vcpu->regs[VCPU_REGS_R8] = regs->r8;
  1725. vcpu->regs[VCPU_REGS_R9] = regs->r9;
  1726. vcpu->regs[VCPU_REGS_R10] = regs->r10;
  1727. vcpu->regs[VCPU_REGS_R11] = regs->r11;
  1728. vcpu->regs[VCPU_REGS_R12] = regs->r12;
  1729. vcpu->regs[VCPU_REGS_R13] = regs->r13;
  1730. vcpu->regs[VCPU_REGS_R14] = regs->r14;
  1731. vcpu->regs[VCPU_REGS_R15] = regs->r15;
  1732. #endif
  1733. vcpu->rip = regs->rip;
  1734. kvm_arch_ops->set_rflags(vcpu, regs->rflags);
  1735. kvm_arch_ops->decache_regs(vcpu);
  1736. vcpu_put(vcpu);
  1737. return 0;
  1738. }
  1739. static void get_segment(struct kvm_vcpu *vcpu,
  1740. struct kvm_segment *var, int seg)
  1741. {
  1742. return kvm_arch_ops->get_segment(vcpu, var, seg);
  1743. }
  1744. static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  1745. struct kvm_sregs *sregs)
  1746. {
  1747. struct descriptor_table dt;
  1748. vcpu_load(vcpu);
  1749. get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  1750. get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  1751. get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  1752. get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  1753. get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  1754. get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  1755. get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  1756. get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  1757. kvm_arch_ops->get_idt(vcpu, &dt);
  1758. sregs->idt.limit = dt.limit;
  1759. sregs->idt.base = dt.base;
  1760. kvm_arch_ops->get_gdt(vcpu, &dt);
  1761. sregs->gdt.limit = dt.limit;
  1762. sregs->gdt.base = dt.base;
  1763. kvm_arch_ops->decache_cr4_guest_bits(vcpu);
  1764. sregs->cr0 = vcpu->cr0;
  1765. sregs->cr2 = vcpu->cr2;
  1766. sregs->cr3 = vcpu->cr3;
  1767. sregs->cr4 = vcpu->cr4;
  1768. sregs->cr8 = get_cr8(vcpu);
  1769. sregs->efer = vcpu->shadow_efer;
  1770. sregs->apic_base = kvm_get_apic_base(vcpu);
  1771. memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
  1772. sizeof sregs->interrupt_bitmap);
  1773. vcpu_put(vcpu);
  1774. return 0;
  1775. }
  1776. static void set_segment(struct kvm_vcpu *vcpu,
  1777. struct kvm_segment *var, int seg)
  1778. {
  1779. return kvm_arch_ops->set_segment(vcpu, var, seg);
  1780. }
  1781. static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  1782. struct kvm_sregs *sregs)
  1783. {
  1784. int mmu_reset_needed = 0;
  1785. int i;
  1786. struct descriptor_table dt;
  1787. vcpu_load(vcpu);
  1788. dt.limit = sregs->idt.limit;
  1789. dt.base = sregs->idt.base;
  1790. kvm_arch_ops->set_idt(vcpu, &dt);
  1791. dt.limit = sregs->gdt.limit;
  1792. dt.base = sregs->gdt.base;
  1793. kvm_arch_ops->set_gdt(vcpu, &dt);
  1794. vcpu->cr2 = sregs->cr2;
  1795. mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
  1796. vcpu->cr3 = sregs->cr3;
  1797. set_cr8(vcpu, sregs->cr8);
  1798. mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
  1799. #ifdef CONFIG_X86_64
  1800. kvm_arch_ops->set_efer(vcpu, sregs->efer);
  1801. #endif
  1802. kvm_set_apic_base(vcpu, sregs->apic_base);
  1803. kvm_arch_ops->decache_cr4_guest_bits(vcpu);
  1804. mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
  1805. kvm_arch_ops->set_cr0(vcpu, sregs->cr0);
  1806. mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
  1807. kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
  1808. if (!is_long_mode(vcpu) && is_pae(vcpu))
  1809. load_pdptrs(vcpu, vcpu->cr3);
  1810. if (mmu_reset_needed)
  1811. kvm_mmu_reset_context(vcpu);
  1812. memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
  1813. sizeof vcpu->irq_pending);
  1814. vcpu->irq_summary = 0;
  1815. for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i)
  1816. if (vcpu->irq_pending[i])
  1817. __set_bit(i, &vcpu->irq_summary);
  1818. set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  1819. set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  1820. set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  1821. set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  1822. set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  1823. set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  1824. set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  1825. set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  1826. vcpu_put(vcpu);
  1827. return 0;
  1828. }
  1829. /*
  1830. * List of msr numbers which we expose to userspace through KVM_GET_MSRS
  1831. * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
  1832. *
  1833. * This list is modified at module load time to reflect the
  1834. * capabilities of the host cpu.
  1835. */
  1836. static u32 msrs_to_save[] = {
  1837. MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  1838. MSR_K6_STAR,
  1839. #ifdef CONFIG_X86_64
  1840. MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
  1841. #endif
  1842. MSR_IA32_TIME_STAMP_COUNTER,
  1843. };
  1844. static unsigned num_msrs_to_save;
  1845. static u32 emulated_msrs[] = {
  1846. MSR_IA32_MISC_ENABLE,
  1847. };
  1848. static __init void kvm_init_msr_list(void)
  1849. {
  1850. u32 dummy[2];
  1851. unsigned i, j;
  1852. for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
  1853. if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
  1854. continue;
  1855. if (j < i)
  1856. msrs_to_save[j] = msrs_to_save[i];
  1857. j++;
  1858. }
  1859. num_msrs_to_save = j;
  1860. }
  1861. /*
  1862. * Adapt set_msr() to msr_io()'s calling convention
  1863. */
  1864. static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
  1865. {
  1866. return kvm_set_msr(vcpu, index, *data);
  1867. }
  1868. /*
  1869. * Read or write a bunch of msrs. All parameters are kernel addresses.
  1870. *
  1871. * @return number of msrs set successfully.
  1872. */
  1873. static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
  1874. struct kvm_msr_entry *entries,
  1875. int (*do_msr)(struct kvm_vcpu *vcpu,
  1876. unsigned index, u64 *data))
  1877. {
  1878. int i;
  1879. vcpu_load(vcpu);
  1880. for (i = 0; i < msrs->nmsrs; ++i)
  1881. if (do_msr(vcpu, entries[i].index, &entries[i].data))
  1882. break;
  1883. vcpu_put(vcpu);
  1884. return i;
  1885. }
  1886. /*
  1887. * Read or write a bunch of msrs. Parameters are user addresses.
  1888. *
  1889. * @return number of msrs set successfully.
  1890. */
  1891. static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
  1892. int (*do_msr)(struct kvm_vcpu *vcpu,
  1893. unsigned index, u64 *data),
  1894. int writeback)
  1895. {
  1896. struct kvm_msrs msrs;
  1897. struct kvm_msr_entry *entries;
  1898. int r, n;
  1899. unsigned size;
  1900. r = -EFAULT;
  1901. if (copy_from_user(&msrs, user_msrs, sizeof msrs))
  1902. goto out;
  1903. r = -E2BIG;
  1904. if (msrs.nmsrs >= MAX_IO_MSRS)
  1905. goto out;
  1906. r = -ENOMEM;
  1907. size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
  1908. entries = vmalloc(size);
  1909. if (!entries)
  1910. goto out;
  1911. r = -EFAULT;
  1912. if (copy_from_user(entries, user_msrs->entries, size))
  1913. goto out_free;
  1914. r = n = __msr_io(vcpu, &msrs, entries, do_msr);
  1915. if (r < 0)
  1916. goto out_free;
  1917. r = -EFAULT;
  1918. if (writeback && copy_to_user(user_msrs->entries, entries, size))
  1919. goto out_free;
  1920. r = n;
  1921. out_free:
  1922. vfree(entries);
  1923. out:
  1924. return r;
  1925. }
  1926. /*
  1927. * Translate a guest virtual address to a guest physical address.
  1928. */
  1929. static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  1930. struct kvm_translation *tr)
  1931. {
  1932. unsigned long vaddr = tr->linear_address;
  1933. gpa_t gpa;
  1934. vcpu_load(vcpu);
  1935. mutex_lock(&vcpu->kvm->lock);
  1936. gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
  1937. tr->physical_address = gpa;
  1938. tr->valid = gpa != UNMAPPED_GVA;
  1939. tr->writeable = 1;
  1940. tr->usermode = 0;
  1941. mutex_unlock(&vcpu->kvm->lock);
  1942. vcpu_put(vcpu);
  1943. return 0;
  1944. }
  1945. static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
  1946. struct kvm_interrupt *irq)
  1947. {
  1948. if (irq->irq < 0 || irq->irq >= 256)
  1949. return -EINVAL;
  1950. if (irqchip_in_kernel(vcpu->kvm))
  1951. return -ENXIO;
  1952. vcpu_load(vcpu);
  1953. set_bit(irq->irq, vcpu->irq_pending);
  1954. set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
  1955. vcpu_put(vcpu);
  1956. return 0;
  1957. }
  1958. static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
  1959. struct kvm_debug_guest *dbg)
  1960. {
  1961. int r;
  1962. vcpu_load(vcpu);
  1963. r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
  1964. vcpu_put(vcpu);
  1965. return r;
  1966. }
  1967. static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
  1968. unsigned long address,
  1969. int *type)
  1970. {
  1971. struct kvm_vcpu *vcpu = vma->vm_file->private_data;
  1972. unsigned long pgoff;
  1973. struct page *page;
  1974. pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  1975. if (pgoff == 0)
  1976. page = virt_to_page(vcpu->run);
  1977. else if (pgoff == KVM_PIO_PAGE_OFFSET)
  1978. page = virt_to_page(vcpu->pio_data);
  1979. else
  1980. return NOPAGE_SIGBUS;
  1981. get_page(page);
  1982. if (type != NULL)
  1983. *type = VM_FAULT_MINOR;
  1984. return page;
  1985. }
  1986. static struct vm_operations_struct kvm_vcpu_vm_ops = {
  1987. .nopage = kvm_vcpu_nopage,
  1988. };
  1989. static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
  1990. {
  1991. vma->vm_ops = &kvm_vcpu_vm_ops;
  1992. return 0;
  1993. }
  1994. static int kvm_vcpu_release(struct inode *inode, struct file *filp)
  1995. {
  1996. struct kvm_vcpu *vcpu = filp->private_data;
  1997. fput(vcpu->kvm->filp);
  1998. return 0;
  1999. }
  2000. static struct file_operations kvm_vcpu_fops = {
  2001. .release = kvm_vcpu_release,
  2002. .unlocked_ioctl = kvm_vcpu_ioctl,
  2003. .compat_ioctl = kvm_vcpu_ioctl,
  2004. .mmap = kvm_vcpu_mmap,
  2005. };
  2006. /*
  2007. * Allocates an inode for the vcpu.
  2008. */
  2009. static int create_vcpu_fd(struct kvm_vcpu *vcpu)
  2010. {
  2011. int fd, r;
  2012. struct inode *inode;
  2013. struct file *file;
  2014. r = anon_inode_getfd(&fd, &inode, &file,
  2015. "kvm-vcpu", &kvm_vcpu_fops, vcpu);
  2016. if (r)
  2017. return r;
  2018. atomic_inc(&vcpu->kvm->filp->f_count);
  2019. return fd;
  2020. }
  2021. /*
  2022. * Creates some virtual cpus. Good luck creating more than one.
  2023. */
  2024. static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
  2025. {
  2026. int r;
  2027. struct kvm_vcpu *vcpu;
  2028. if (!valid_vcpu(n))
  2029. return -EINVAL;
  2030. vcpu = kvm_arch_ops->vcpu_create(kvm, n);
  2031. if (IS_ERR(vcpu))
  2032. return PTR_ERR(vcpu);
  2033. preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
  2034. /* We do fxsave: this must be aligned. */
  2035. BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF);
  2036. vcpu_load(vcpu);
  2037. r = kvm_mmu_setup(vcpu);
  2038. vcpu_put(vcpu);
  2039. if (r < 0)
  2040. goto free_vcpu;
  2041. mutex_lock(&kvm->lock);
  2042. if (kvm->vcpus[n]) {
  2043. r = -EEXIST;
  2044. mutex_unlock(&kvm->lock);
  2045. goto mmu_unload;
  2046. }
  2047. kvm->vcpus[n] = vcpu;
  2048. mutex_unlock(&kvm->lock);
  2049. /* Now it's all set up, let userspace reach it */
  2050. r = create_vcpu_fd(vcpu);
  2051. if (r < 0)
  2052. goto unlink;
  2053. return r;
  2054. unlink:
  2055. mutex_lock(&kvm->lock);
  2056. kvm->vcpus[n] = NULL;
  2057. mutex_unlock(&kvm->lock);
  2058. mmu_unload:
  2059. vcpu_load(vcpu);
  2060. kvm_mmu_unload(vcpu);
  2061. vcpu_put(vcpu);
  2062. free_vcpu:
  2063. kvm_arch_ops->vcpu_free(vcpu);
  2064. return r;
  2065. }
  2066. static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
  2067. {
  2068. u64 efer;
  2069. int i;
  2070. struct kvm_cpuid_entry *e, *entry;
  2071. rdmsrl(MSR_EFER, efer);
  2072. entry = NULL;
  2073. for (i = 0; i < vcpu->cpuid_nent; ++i) {
  2074. e = &vcpu->cpuid_entries[i];
  2075. if (e->function == 0x80000001) {
  2076. entry = e;
  2077. break;
  2078. }
  2079. }
  2080. if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) {
  2081. entry->edx &= ~(1 << 20);
  2082. printk(KERN_INFO "kvm: guest NX capability removed\n");
  2083. }
  2084. }
  2085. static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
  2086. struct kvm_cpuid *cpuid,
  2087. struct kvm_cpuid_entry __user *entries)
  2088. {
  2089. int r;
  2090. r = -E2BIG;
  2091. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  2092. goto out;
  2093. r = -EFAULT;
  2094. if (copy_from_user(&vcpu->cpuid_entries, entries,
  2095. cpuid->nent * sizeof(struct kvm_cpuid_entry)))
  2096. goto out;
  2097. vcpu->cpuid_nent = cpuid->nent;
  2098. cpuid_fix_nx_cap(vcpu);
  2099. return 0;
  2100. out:
  2101. return r;
  2102. }
  2103. static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
  2104. {
  2105. if (sigset) {
  2106. sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
  2107. vcpu->sigset_active = 1;
  2108. vcpu->sigset = *sigset;
  2109. } else
  2110. vcpu->sigset_active = 0;
  2111. return 0;
  2112. }
  2113. /*
  2114. * fxsave fpu state. Taken from x86_64/processor.h. To be killed when
  2115. * we have asm/x86/processor.h
  2116. */
  2117. struct fxsave {
  2118. u16 cwd;
  2119. u16 swd;
  2120. u16 twd;
  2121. u16 fop;
  2122. u64 rip;
  2123. u64 rdp;
  2124. u32 mxcsr;
  2125. u32 mxcsr_mask;
  2126. u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */
  2127. #ifdef CONFIG_X86_64
  2128. u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */
  2129. #else
  2130. u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */
  2131. #endif
  2132. };
  2133. static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  2134. {
  2135. struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image;
  2136. vcpu_load(vcpu);
  2137. memcpy(fpu->fpr, fxsave->st_space, 128);
  2138. fpu->fcw = fxsave->cwd;
  2139. fpu->fsw = fxsave->swd;
  2140. fpu->ftwx = fxsave->twd;
  2141. fpu->last_opcode = fxsave->fop;
  2142. fpu->last_ip = fxsave->rip;
  2143. fpu->last_dp = fxsave->rdp;
  2144. memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
  2145. vcpu_put(vcpu);
  2146. return 0;
  2147. }
  2148. static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  2149. {
  2150. struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image;
  2151. vcpu_load(vcpu);
  2152. memcpy(fxsave->st_space, fpu->fpr, 128);
  2153. fxsave->cwd = fpu->fcw;
  2154. fxsave->swd = fpu->fsw;
  2155. fxsave->twd = fpu->ftwx;
  2156. fxsave->fop = fpu->last_opcode;
  2157. fxsave->rip = fpu->last_ip;
  2158. fxsave->rdp = fpu->last_dp;
  2159. memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
  2160. vcpu_put(vcpu);
  2161. return 0;
  2162. }
  2163. static long kvm_vcpu_ioctl(struct file *filp,
  2164. unsigned int ioctl, unsigned long arg)
  2165. {
  2166. struct kvm_vcpu *vcpu = filp->private_data;
  2167. void __user *argp = (void __user *)arg;
  2168. int r = -EINVAL;
  2169. switch (ioctl) {
  2170. case KVM_RUN:
  2171. r = -EINVAL;
  2172. if (arg)
  2173. goto out;
  2174. r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
  2175. break;
  2176. case KVM_GET_REGS: {
  2177. struct kvm_regs kvm_regs;
  2178. memset(&kvm_regs, 0, sizeof kvm_regs);
  2179. r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
  2180. if (r)
  2181. goto out;
  2182. r = -EFAULT;
  2183. if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
  2184. goto out;
  2185. r = 0;
  2186. break;
  2187. }
  2188. case KVM_SET_REGS: {
  2189. struct kvm_regs kvm_regs;
  2190. r = -EFAULT;
  2191. if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
  2192. goto out;
  2193. r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
  2194. if (r)
  2195. goto out;
  2196. r = 0;
  2197. break;
  2198. }
  2199. case KVM_GET_SREGS: {
  2200. struct kvm_sregs kvm_sregs;
  2201. memset(&kvm_sregs, 0, sizeof kvm_sregs);
  2202. r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
  2203. if (r)
  2204. goto out;
  2205. r = -EFAULT;
  2206. if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
  2207. goto out;
  2208. r = 0;
  2209. break;
  2210. }
  2211. case KVM_SET_SREGS: {
  2212. struct kvm_sregs kvm_sregs;
  2213. r = -EFAULT;
  2214. if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
  2215. goto out;
  2216. r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
  2217. if (r)
  2218. goto out;
  2219. r = 0;
  2220. break;
  2221. }
  2222. case KVM_TRANSLATE: {
  2223. struct kvm_translation tr;
  2224. r = -EFAULT;
  2225. if (copy_from_user(&tr, argp, sizeof tr))
  2226. goto out;
  2227. r = kvm_vcpu_ioctl_translate(vcpu, &tr);
  2228. if (r)
  2229. goto out;
  2230. r = -EFAULT;
  2231. if (copy_to_user(argp, &tr, sizeof tr))
  2232. goto out;
  2233. r = 0;
  2234. break;
  2235. }
  2236. case KVM_INTERRUPT: {
  2237. struct kvm_interrupt irq;
  2238. r = -EFAULT;
  2239. if (copy_from_user(&irq, argp, sizeof irq))
  2240. goto out;
  2241. r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
  2242. if (r)
  2243. goto out;
  2244. r = 0;
  2245. break;
  2246. }
  2247. case KVM_DEBUG_GUEST: {
  2248. struct kvm_debug_guest dbg;
  2249. r = -EFAULT;
  2250. if (copy_from_user(&dbg, argp, sizeof dbg))
  2251. goto out;
  2252. r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
  2253. if (r)
  2254. goto out;
  2255. r = 0;
  2256. break;
  2257. }
  2258. case KVM_GET_MSRS:
  2259. r = msr_io(vcpu, argp, kvm_get_msr, 1);
  2260. break;
  2261. case KVM_SET_MSRS:
  2262. r = msr_io(vcpu, argp, do_set_msr, 0);
  2263. break;
  2264. case KVM_SET_CPUID: {
  2265. struct kvm_cpuid __user *cpuid_arg = argp;
  2266. struct kvm_cpuid cpuid;
  2267. r = -EFAULT;
  2268. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  2269. goto out;
  2270. r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
  2271. if (r)
  2272. goto out;
  2273. break;
  2274. }
  2275. case KVM_SET_SIGNAL_MASK: {
  2276. struct kvm_signal_mask __user *sigmask_arg = argp;
  2277. struct kvm_signal_mask kvm_sigmask;
  2278. sigset_t sigset, *p;
  2279. p = NULL;
  2280. if (argp) {
  2281. r = -EFAULT;
  2282. if (copy_from_user(&kvm_sigmask, argp,
  2283. sizeof kvm_sigmask))
  2284. goto out;
  2285. r = -EINVAL;
  2286. if (kvm_sigmask.len != sizeof sigset)
  2287. goto out;
  2288. r = -EFAULT;
  2289. if (copy_from_user(&sigset, sigmask_arg->sigset,
  2290. sizeof sigset))
  2291. goto out;
  2292. p = &sigset;
  2293. }
  2294. r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
  2295. break;
  2296. }
  2297. case KVM_GET_FPU: {
  2298. struct kvm_fpu fpu;
  2299. memset(&fpu, 0, sizeof fpu);
  2300. r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu);
  2301. if (r)
  2302. goto out;
  2303. r = -EFAULT;
  2304. if (copy_to_user(argp, &fpu, sizeof fpu))
  2305. goto out;
  2306. r = 0;
  2307. break;
  2308. }
  2309. case KVM_SET_FPU: {
  2310. struct kvm_fpu fpu;
  2311. r = -EFAULT;
  2312. if (copy_from_user(&fpu, argp, sizeof fpu))
  2313. goto out;
  2314. r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu);
  2315. if (r)
  2316. goto out;
  2317. r = 0;
  2318. break;
  2319. }
  2320. default:
  2321. ;
  2322. }
  2323. out:
  2324. return r;
  2325. }
  2326. static long kvm_vm_ioctl(struct file *filp,
  2327. unsigned int ioctl, unsigned long arg)
  2328. {
  2329. struct kvm *kvm = filp->private_data;
  2330. void __user *argp = (void __user *)arg;
  2331. int r = -EINVAL;
  2332. switch (ioctl) {
  2333. case KVM_CREATE_VCPU:
  2334. r = kvm_vm_ioctl_create_vcpu(kvm, arg);
  2335. if (r < 0)
  2336. goto out;
  2337. break;
  2338. case KVM_SET_MEMORY_REGION: {
  2339. struct kvm_memory_region kvm_mem;
  2340. r = -EFAULT;
  2341. if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
  2342. goto out;
  2343. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
  2344. if (r)
  2345. goto out;
  2346. break;
  2347. }
  2348. case KVM_GET_DIRTY_LOG: {
  2349. struct kvm_dirty_log log;
  2350. r = -EFAULT;
  2351. if (copy_from_user(&log, argp, sizeof log))
  2352. goto out;
  2353. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  2354. if (r)
  2355. goto out;
  2356. break;
  2357. }
  2358. case KVM_SET_MEMORY_ALIAS: {
  2359. struct kvm_memory_alias alias;
  2360. r = -EFAULT;
  2361. if (copy_from_user(&alias, argp, sizeof alias))
  2362. goto out;
  2363. r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
  2364. if (r)
  2365. goto out;
  2366. break;
  2367. }
  2368. case KVM_CREATE_IRQCHIP:
  2369. r = -ENOMEM;
  2370. kvm->vpic = kvm_create_pic(kvm);
  2371. if (kvm->vpic) {
  2372. r = kvm_ioapic_init(kvm);
  2373. if (r) {
  2374. kfree(kvm->vpic);
  2375. kvm->vpic = NULL;
  2376. goto out;
  2377. }
  2378. }
  2379. else
  2380. goto out;
  2381. break;
  2382. case KVM_IRQ_LINE: {
  2383. struct kvm_irq_level irq_event;
  2384. r = -EFAULT;
  2385. if (copy_from_user(&irq_event, argp, sizeof irq_event))
  2386. goto out;
  2387. if (irqchip_in_kernel(kvm)) {
  2388. if (irq_event.irq < 16)
  2389. kvm_pic_set_irq(pic_irqchip(kvm),
  2390. irq_event.irq,
  2391. irq_event.level);
  2392. kvm_ioapic_set_irq(kvm->vioapic,
  2393. irq_event.irq,
  2394. irq_event.level);
  2395. r = 0;
  2396. }
  2397. break;
  2398. }
  2399. default:
  2400. ;
  2401. }
  2402. out:
  2403. return r;
  2404. }
  2405. static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
  2406. unsigned long address,
  2407. int *type)
  2408. {
  2409. struct kvm *kvm = vma->vm_file->private_data;
  2410. unsigned long pgoff;
  2411. struct page *page;
  2412. pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  2413. page = gfn_to_page(kvm, pgoff);
  2414. if (!page)
  2415. return NOPAGE_SIGBUS;
  2416. get_page(page);
  2417. if (type != NULL)
  2418. *type = VM_FAULT_MINOR;
  2419. return page;
  2420. }
  2421. static struct vm_operations_struct kvm_vm_vm_ops = {
  2422. .nopage = kvm_vm_nopage,
  2423. };
  2424. static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
  2425. {
  2426. vma->vm_ops = &kvm_vm_vm_ops;
  2427. return 0;
  2428. }
  2429. static struct file_operations kvm_vm_fops = {
  2430. .release = kvm_vm_release,
  2431. .unlocked_ioctl = kvm_vm_ioctl,
  2432. .compat_ioctl = kvm_vm_ioctl,
  2433. .mmap = kvm_vm_mmap,
  2434. };
  2435. static int kvm_dev_ioctl_create_vm(void)
  2436. {
  2437. int fd, r;
  2438. struct inode *inode;
  2439. struct file *file;
  2440. struct kvm *kvm;
  2441. kvm = kvm_create_vm();
  2442. if (IS_ERR(kvm))
  2443. return PTR_ERR(kvm);
  2444. r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
  2445. if (r) {
  2446. kvm_destroy_vm(kvm);
  2447. return r;
  2448. }
  2449. kvm->filp = file;
  2450. return fd;
  2451. }
  2452. static long kvm_dev_ioctl(struct file *filp,
  2453. unsigned int ioctl, unsigned long arg)
  2454. {
  2455. void __user *argp = (void __user *)arg;
  2456. long r = -EINVAL;
  2457. switch (ioctl) {
  2458. case KVM_GET_API_VERSION:
  2459. r = -EINVAL;
  2460. if (arg)
  2461. goto out;
  2462. r = KVM_API_VERSION;
  2463. break;
  2464. case KVM_CREATE_VM:
  2465. r = -EINVAL;
  2466. if (arg)
  2467. goto out;
  2468. r = kvm_dev_ioctl_create_vm();
  2469. break;
  2470. case KVM_GET_MSR_INDEX_LIST: {
  2471. struct kvm_msr_list __user *user_msr_list = argp;
  2472. struct kvm_msr_list msr_list;
  2473. unsigned n;
  2474. r = -EFAULT;
  2475. if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
  2476. goto out;
  2477. n = msr_list.nmsrs;
  2478. msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
  2479. if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
  2480. goto out;
  2481. r = -E2BIG;
  2482. if (n < num_msrs_to_save)
  2483. goto out;
  2484. r = -EFAULT;
  2485. if (copy_to_user(user_msr_list->indices, &msrs_to_save,
  2486. num_msrs_to_save * sizeof(u32)))
  2487. goto out;
  2488. if (copy_to_user(user_msr_list->indices
  2489. + num_msrs_to_save * sizeof(u32),
  2490. &emulated_msrs,
  2491. ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
  2492. goto out;
  2493. r = 0;
  2494. break;
  2495. }
  2496. case KVM_CHECK_EXTENSION: {
  2497. int ext = (long)argp;
  2498. switch (ext) {
  2499. case KVM_CAP_IRQCHIP:
  2500. r = 1;
  2501. break;
  2502. default:
  2503. r = 0;
  2504. break;
  2505. }
  2506. break;
  2507. }
  2508. case KVM_GET_VCPU_MMAP_SIZE:
  2509. r = -EINVAL;
  2510. if (arg)
  2511. goto out;
  2512. r = 2 * PAGE_SIZE;
  2513. break;
  2514. default:
  2515. ;
  2516. }
  2517. out:
  2518. return r;
  2519. }
  2520. static struct file_operations kvm_chardev_ops = {
  2521. .unlocked_ioctl = kvm_dev_ioctl,
  2522. .compat_ioctl = kvm_dev_ioctl,
  2523. };
  2524. static struct miscdevice kvm_dev = {
  2525. KVM_MINOR,
  2526. "kvm",
  2527. &kvm_chardev_ops,
  2528. };
  2529. /*
  2530. * Make sure that a cpu that is being hot-unplugged does not have any vcpus
  2531. * cached on it.
  2532. */
  2533. static void decache_vcpus_on_cpu(int cpu)
  2534. {
  2535. struct kvm *vm;
  2536. struct kvm_vcpu *vcpu;
  2537. int i;
  2538. spin_lock(&kvm_lock);
  2539. list_for_each_entry(vm, &vm_list, vm_list)
  2540. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  2541. vcpu = vm->vcpus[i];
  2542. if (!vcpu)
  2543. continue;
  2544. /*
  2545. * If the vcpu is locked, then it is running on some
  2546. * other cpu and therefore it is not cached on the
  2547. * cpu in question.
  2548. *
  2549. * If it's not locked, check the last cpu it executed
  2550. * on.
  2551. */
  2552. if (mutex_trylock(&vcpu->mutex)) {
  2553. if (vcpu->cpu == cpu) {
  2554. kvm_arch_ops->vcpu_decache(vcpu);
  2555. vcpu->cpu = -1;
  2556. }
  2557. mutex_unlock(&vcpu->mutex);
  2558. }
  2559. }
  2560. spin_unlock(&kvm_lock);
  2561. }
  2562. static void hardware_enable(void *junk)
  2563. {
  2564. int cpu = raw_smp_processor_id();
  2565. if (cpu_isset(cpu, cpus_hardware_enabled))
  2566. return;
  2567. cpu_set(cpu, cpus_hardware_enabled);
  2568. kvm_arch_ops->hardware_enable(NULL);
  2569. }
  2570. static void hardware_disable(void *junk)
  2571. {
  2572. int cpu = raw_smp_processor_id();
  2573. if (!cpu_isset(cpu, cpus_hardware_enabled))
  2574. return;
  2575. cpu_clear(cpu, cpus_hardware_enabled);
  2576. decache_vcpus_on_cpu(cpu);
  2577. kvm_arch_ops->hardware_disable(NULL);
  2578. }
  2579. static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  2580. void *v)
  2581. {
  2582. int cpu = (long)v;
  2583. switch (val) {
  2584. case CPU_DYING:
  2585. case CPU_DYING_FROZEN:
  2586. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  2587. cpu);
  2588. hardware_disable(NULL);
  2589. break;
  2590. case CPU_UP_CANCELED:
  2591. case CPU_UP_CANCELED_FROZEN:
  2592. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  2593. cpu);
  2594. smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
  2595. break;
  2596. case CPU_ONLINE:
  2597. case CPU_ONLINE_FROZEN:
  2598. printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
  2599. cpu);
  2600. smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
  2601. break;
  2602. }
  2603. return NOTIFY_OK;
  2604. }
  2605. static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
  2606. void *v)
  2607. {
  2608. if (val == SYS_RESTART) {
  2609. /*
  2610. * Some (well, at least mine) BIOSes hang on reboot if
  2611. * in vmx root mode.
  2612. */
  2613. printk(KERN_INFO "kvm: exiting hardware virtualization\n");
  2614. on_each_cpu(hardware_disable, NULL, 0, 1);
  2615. }
  2616. return NOTIFY_OK;
  2617. }
  2618. static struct notifier_block kvm_reboot_notifier = {
  2619. .notifier_call = kvm_reboot,
  2620. .priority = 0,
  2621. };
  2622. void kvm_io_bus_init(struct kvm_io_bus *bus)
  2623. {
  2624. memset(bus, 0, sizeof(*bus));
  2625. }
  2626. void kvm_io_bus_destroy(struct kvm_io_bus *bus)
  2627. {
  2628. int i;
  2629. for (i = 0; i < bus->dev_count; i++) {
  2630. struct kvm_io_device *pos = bus->devs[i];
  2631. kvm_iodevice_destructor(pos);
  2632. }
  2633. }
  2634. struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
  2635. {
  2636. int i;
  2637. for (i = 0; i < bus->dev_count; i++) {
  2638. struct kvm_io_device *pos = bus->devs[i];
  2639. if (pos->in_range(pos, addr))
  2640. return pos;
  2641. }
  2642. return NULL;
  2643. }
  2644. void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
  2645. {
  2646. BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
  2647. bus->devs[bus->dev_count++] = dev;
  2648. }
  2649. static struct notifier_block kvm_cpu_notifier = {
  2650. .notifier_call = kvm_cpu_hotplug,
  2651. .priority = 20, /* must be > scheduler priority */
  2652. };
  2653. static u64 stat_get(void *_offset)
  2654. {
  2655. unsigned offset = (long)_offset;
  2656. u64 total = 0;
  2657. struct kvm *kvm;
  2658. struct kvm_vcpu *vcpu;
  2659. int i;
  2660. spin_lock(&kvm_lock);
  2661. list_for_each_entry(kvm, &vm_list, vm_list)
  2662. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  2663. vcpu = kvm->vcpus[i];
  2664. if (vcpu)
  2665. total += *(u32 *)((void *)vcpu + offset);
  2666. }
  2667. spin_unlock(&kvm_lock);
  2668. return total;
  2669. }
  2670. DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n");
  2671. static __init void kvm_init_debug(void)
  2672. {
  2673. struct kvm_stats_debugfs_item *p;
  2674. debugfs_dir = debugfs_create_dir("kvm", NULL);
  2675. for (p = debugfs_entries; p->name; ++p)
  2676. p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
  2677. (void *)(long)p->offset,
  2678. &stat_fops);
  2679. }
  2680. static void kvm_exit_debug(void)
  2681. {
  2682. struct kvm_stats_debugfs_item *p;
  2683. for (p = debugfs_entries; p->name; ++p)
  2684. debugfs_remove(p->dentry);
  2685. debugfs_remove(debugfs_dir);
  2686. }
  2687. static int kvm_suspend(struct sys_device *dev, pm_message_t state)
  2688. {
  2689. hardware_disable(NULL);
  2690. return 0;
  2691. }
  2692. static int kvm_resume(struct sys_device *dev)
  2693. {
  2694. hardware_enable(NULL);
  2695. return 0;
  2696. }
  2697. static struct sysdev_class kvm_sysdev_class = {
  2698. set_kset_name("kvm"),
  2699. .suspend = kvm_suspend,
  2700. .resume = kvm_resume,
  2701. };
  2702. static struct sys_device kvm_sysdev = {
  2703. .id = 0,
  2704. .cls = &kvm_sysdev_class,
  2705. };
  2706. hpa_t bad_page_address;
  2707. static inline
  2708. struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
  2709. {
  2710. return container_of(pn, struct kvm_vcpu, preempt_notifier);
  2711. }
  2712. static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
  2713. {
  2714. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  2715. kvm_arch_ops->vcpu_load(vcpu, cpu);
  2716. }
  2717. static void kvm_sched_out(struct preempt_notifier *pn,
  2718. struct task_struct *next)
  2719. {
  2720. struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
  2721. kvm_arch_ops->vcpu_put(vcpu);
  2722. }
  2723. int kvm_init_arch(struct kvm_arch_ops *ops, unsigned int vcpu_size,
  2724. struct module *module)
  2725. {
  2726. int r;
  2727. int cpu;
  2728. if (kvm_arch_ops) {
  2729. printk(KERN_ERR "kvm: already loaded the other module\n");
  2730. return -EEXIST;
  2731. }
  2732. if (!ops->cpu_has_kvm_support()) {
  2733. printk(KERN_ERR "kvm: no hardware support\n");
  2734. return -EOPNOTSUPP;
  2735. }
  2736. if (ops->disabled_by_bios()) {
  2737. printk(KERN_ERR "kvm: disabled by bios\n");
  2738. return -EOPNOTSUPP;
  2739. }
  2740. kvm_arch_ops = ops;
  2741. r = kvm_arch_ops->hardware_setup();
  2742. if (r < 0)
  2743. goto out;
  2744. for_each_online_cpu(cpu) {
  2745. smp_call_function_single(cpu,
  2746. kvm_arch_ops->check_processor_compatibility,
  2747. &r, 0, 1);
  2748. if (r < 0)
  2749. goto out_free_0;
  2750. }
  2751. on_each_cpu(hardware_enable, NULL, 0, 1);
  2752. r = register_cpu_notifier(&kvm_cpu_notifier);
  2753. if (r)
  2754. goto out_free_1;
  2755. register_reboot_notifier(&kvm_reboot_notifier);
  2756. r = sysdev_class_register(&kvm_sysdev_class);
  2757. if (r)
  2758. goto out_free_2;
  2759. r = sysdev_register(&kvm_sysdev);
  2760. if (r)
  2761. goto out_free_3;
  2762. /* A kmem cache lets us meet the alignment requirements of fx_save. */
  2763. kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
  2764. __alignof__(struct kvm_vcpu), 0, 0);
  2765. if (!kvm_vcpu_cache) {
  2766. r = -ENOMEM;
  2767. goto out_free_4;
  2768. }
  2769. kvm_chardev_ops.owner = module;
  2770. r = misc_register(&kvm_dev);
  2771. if (r) {
  2772. printk (KERN_ERR "kvm: misc device register failed\n");
  2773. goto out_free;
  2774. }
  2775. kvm_preempt_ops.sched_in = kvm_sched_in;
  2776. kvm_preempt_ops.sched_out = kvm_sched_out;
  2777. return r;
  2778. out_free:
  2779. kmem_cache_destroy(kvm_vcpu_cache);
  2780. out_free_4:
  2781. sysdev_unregister(&kvm_sysdev);
  2782. out_free_3:
  2783. sysdev_class_unregister(&kvm_sysdev_class);
  2784. out_free_2:
  2785. unregister_reboot_notifier(&kvm_reboot_notifier);
  2786. unregister_cpu_notifier(&kvm_cpu_notifier);
  2787. out_free_1:
  2788. on_each_cpu(hardware_disable, NULL, 0, 1);
  2789. out_free_0:
  2790. kvm_arch_ops->hardware_unsetup();
  2791. out:
  2792. kvm_arch_ops = NULL;
  2793. return r;
  2794. }
  2795. void kvm_exit_arch(void)
  2796. {
  2797. misc_deregister(&kvm_dev);
  2798. kmem_cache_destroy(kvm_vcpu_cache);
  2799. sysdev_unregister(&kvm_sysdev);
  2800. sysdev_class_unregister(&kvm_sysdev_class);
  2801. unregister_reboot_notifier(&kvm_reboot_notifier);
  2802. unregister_cpu_notifier(&kvm_cpu_notifier);
  2803. on_each_cpu(hardware_disable, NULL, 0, 1);
  2804. kvm_arch_ops->hardware_unsetup();
  2805. kvm_arch_ops = NULL;
  2806. }
  2807. static __init int kvm_init(void)
  2808. {
  2809. static struct page *bad_page;
  2810. int r;
  2811. r = kvm_mmu_module_init();
  2812. if (r)
  2813. goto out4;
  2814. kvm_init_debug();
  2815. kvm_init_msr_list();
  2816. if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
  2817. r = -ENOMEM;
  2818. goto out;
  2819. }
  2820. bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
  2821. memset(__va(bad_page_address), 0, PAGE_SIZE);
  2822. return 0;
  2823. out:
  2824. kvm_exit_debug();
  2825. kvm_mmu_module_exit();
  2826. out4:
  2827. return r;
  2828. }
  2829. static __exit void kvm_exit(void)
  2830. {
  2831. kvm_exit_debug();
  2832. __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
  2833. kvm_mmu_module_exit();
  2834. }
  2835. module_init(kvm_init)
  2836. module_exit(kvm_exit)
  2837. EXPORT_SYMBOL_GPL(kvm_init_arch);
  2838. EXPORT_SYMBOL_GPL(kvm_exit_arch);