kvm_main.c 70 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. *
  9. * Authors:
  10. * Avi Kivity <avi@qumranet.com>
  11. * Yaniv Kamay <yaniv@qumranet.com>
  12. *
  13. * This work is licensed under the terms of the GNU GPL, version 2. See
  14. * the COPYING file in the top-level directory.
  15. *
  16. */
  17. #include "kvm.h"
  18. #include "x86_emulate.h"
  19. #include "segment_descriptor.h"
  20. #include <linux/kvm.h>
  21. #include <linux/module.h>
  22. #include <linux/errno.h>
  23. #include <linux/percpu.h>
  24. #include <linux/gfp.h>
  25. #include <linux/mm.h>
  26. #include <linux/miscdevice.h>
  27. #include <linux/vmalloc.h>
  28. #include <linux/reboot.h>
  29. #include <linux/debugfs.h>
  30. #include <linux/highmem.h>
  31. #include <linux/file.h>
  32. #include <linux/sysdev.h>
  33. #include <linux/cpu.h>
  34. #include <linux/sched.h>
  35. #include <linux/cpumask.h>
  36. #include <linux/smp.h>
  37. #include <linux/anon_inodes.h>
  38. #include <asm/processor.h>
  39. #include <asm/msr.h>
  40. #include <asm/io.h>
  41. #include <asm/uaccess.h>
  42. #include <asm/desc.h>
  43. MODULE_AUTHOR("Qumranet");
  44. MODULE_LICENSE("GPL");
  45. static DEFINE_SPINLOCK(kvm_lock);
  46. static LIST_HEAD(vm_list);
  47. static cpumask_t cpus_hardware_enabled;
  48. struct kvm_arch_ops *kvm_arch_ops;
  49. static void hardware_disable(void *ignored);
  50. #define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x)
  51. static struct kvm_stats_debugfs_item {
  52. const char *name;
  53. int offset;
  54. struct dentry *dentry;
  55. } debugfs_entries[] = {
  56. { "pf_fixed", STAT_OFFSET(pf_fixed) },
  57. { "pf_guest", STAT_OFFSET(pf_guest) },
  58. { "tlb_flush", STAT_OFFSET(tlb_flush) },
  59. { "invlpg", STAT_OFFSET(invlpg) },
  60. { "exits", STAT_OFFSET(exits) },
  61. { "io_exits", STAT_OFFSET(io_exits) },
  62. { "mmio_exits", STAT_OFFSET(mmio_exits) },
  63. { "signal_exits", STAT_OFFSET(signal_exits) },
  64. { "irq_window", STAT_OFFSET(irq_window_exits) },
  65. { "halt_exits", STAT_OFFSET(halt_exits) },
  66. { "request_irq", STAT_OFFSET(request_irq_exits) },
  67. { "irq_exits", STAT_OFFSET(irq_exits) },
  68. { "light_exits", STAT_OFFSET(light_exits) },
  69. { "efer_reload", STAT_OFFSET(efer_reload) },
  70. { NULL }
  71. };
  72. static struct dentry *debugfs_dir;
  73. #define MAX_IO_MSRS 256
  74. #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
  75. #define LMSW_GUEST_MASK 0x0eULL
  76. #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
  77. #define CR8_RESEVED_BITS (~0x0fULL)
  78. #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  79. #ifdef CONFIG_X86_64
  80. // LDT or TSS descriptor in the GDT. 16 bytes.
  81. struct segment_descriptor_64 {
  82. struct segment_descriptor s;
  83. u32 base_higher;
  84. u32 pad_zero;
  85. };
  86. #endif
  87. static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  88. unsigned long arg);
  89. unsigned long segment_base(u16 selector)
  90. {
  91. struct descriptor_table gdt;
  92. struct segment_descriptor *d;
  93. unsigned long table_base;
  94. typedef unsigned long ul;
  95. unsigned long v;
  96. if (selector == 0)
  97. return 0;
  98. asm ("sgdt %0" : "=m"(gdt));
  99. table_base = gdt.base;
  100. if (selector & 4) { /* from ldt */
  101. u16 ldt_selector;
  102. asm ("sldt %0" : "=g"(ldt_selector));
  103. table_base = segment_base(ldt_selector);
  104. }
  105. d = (struct segment_descriptor *)(table_base + (selector & ~7));
  106. v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
  107. #ifdef CONFIG_X86_64
  108. if (d->system == 0
  109. && (d->type == 2 || d->type == 9 || d->type == 11))
  110. v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
  111. #endif
  112. return v;
  113. }
  114. EXPORT_SYMBOL_GPL(segment_base);
  115. static inline int valid_vcpu(int n)
  116. {
  117. return likely(n >= 0 && n < KVM_MAX_VCPUS);
  118. }
  119. int kvm_read_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
  120. void *dest)
  121. {
  122. unsigned char *host_buf = dest;
  123. unsigned long req_size = size;
  124. while (size) {
  125. hpa_t paddr;
  126. unsigned now;
  127. unsigned offset;
  128. hva_t guest_buf;
  129. paddr = gva_to_hpa(vcpu, addr);
  130. if (is_error_hpa(paddr))
  131. break;
  132. guest_buf = (hva_t)kmap_atomic(
  133. pfn_to_page(paddr >> PAGE_SHIFT),
  134. KM_USER0);
  135. offset = addr & ~PAGE_MASK;
  136. guest_buf |= offset;
  137. now = min(size, PAGE_SIZE - offset);
  138. memcpy(host_buf, (void*)guest_buf, now);
  139. host_buf += now;
  140. addr += now;
  141. size -= now;
  142. kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
  143. }
  144. return req_size - size;
  145. }
  146. EXPORT_SYMBOL_GPL(kvm_read_guest);
  147. int kvm_write_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
  148. void *data)
  149. {
  150. unsigned char *host_buf = data;
  151. unsigned long req_size = size;
  152. while (size) {
  153. hpa_t paddr;
  154. unsigned now;
  155. unsigned offset;
  156. hva_t guest_buf;
  157. gfn_t gfn;
  158. paddr = gva_to_hpa(vcpu, addr);
  159. if (is_error_hpa(paddr))
  160. break;
  161. gfn = vcpu->mmu.gva_to_gpa(vcpu, addr) >> PAGE_SHIFT;
  162. mark_page_dirty(vcpu->kvm, gfn);
  163. guest_buf = (hva_t)kmap_atomic(
  164. pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
  165. offset = addr & ~PAGE_MASK;
  166. guest_buf |= offset;
  167. now = min(size, PAGE_SIZE - offset);
  168. memcpy((void*)guest_buf, host_buf, now);
  169. host_buf += now;
  170. addr += now;
  171. size -= now;
  172. kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
  173. }
  174. return req_size - size;
  175. }
  176. EXPORT_SYMBOL_GPL(kvm_write_guest);
  177. void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
  178. {
  179. if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
  180. return;
  181. vcpu->guest_fpu_loaded = 1;
  182. fx_save(vcpu->host_fx_image);
  183. fx_restore(vcpu->guest_fx_image);
  184. }
  185. EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
  186. void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
  187. {
  188. if (!vcpu->guest_fpu_loaded)
  189. return;
  190. vcpu->guest_fpu_loaded = 0;
  191. fx_save(vcpu->guest_fx_image);
  192. fx_restore(vcpu->host_fx_image);
  193. }
  194. EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
  195. /*
  196. * Switches to specified vcpu, until a matching vcpu_put()
  197. */
  198. static void vcpu_load(struct kvm_vcpu *vcpu)
  199. {
  200. mutex_lock(&vcpu->mutex);
  201. kvm_arch_ops->vcpu_load(vcpu);
  202. }
  203. static void vcpu_put(struct kvm_vcpu *vcpu)
  204. {
  205. kvm_arch_ops->vcpu_put(vcpu);
  206. mutex_unlock(&vcpu->mutex);
  207. }
  208. static void ack_flush(void *_completed)
  209. {
  210. atomic_t *completed = _completed;
  211. atomic_inc(completed);
  212. }
  213. void kvm_flush_remote_tlbs(struct kvm *kvm)
  214. {
  215. int i, cpu, needed;
  216. cpumask_t cpus;
  217. struct kvm_vcpu *vcpu;
  218. atomic_t completed;
  219. atomic_set(&completed, 0);
  220. cpus_clear(cpus);
  221. needed = 0;
  222. for (i = 0; i < kvm->nvcpus; ++i) {
  223. vcpu = &kvm->vcpus[i];
  224. if (test_and_set_bit(KVM_TLB_FLUSH, &vcpu->requests))
  225. continue;
  226. cpu = vcpu->cpu;
  227. if (cpu != -1 && cpu != raw_smp_processor_id())
  228. if (!cpu_isset(cpu, cpus)) {
  229. cpu_set(cpu, cpus);
  230. ++needed;
  231. }
  232. }
  233. /*
  234. * We really want smp_call_function_mask() here. But that's not
  235. * available, so ipi all cpus in parallel and wait for them
  236. * to complete.
  237. */
  238. for (cpu = first_cpu(cpus); cpu != NR_CPUS; cpu = next_cpu(cpu, cpus))
  239. smp_call_function_single(cpu, ack_flush, &completed, 1, 0);
  240. while (atomic_read(&completed) != needed) {
  241. cpu_relax();
  242. barrier();
  243. }
  244. }
  245. static struct kvm *kvm_create_vm(void)
  246. {
  247. struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
  248. int i;
  249. if (!kvm)
  250. return ERR_PTR(-ENOMEM);
  251. kvm_io_bus_init(&kvm->pio_bus);
  252. spin_lock_init(&kvm->lock);
  253. INIT_LIST_HEAD(&kvm->active_mmu_pages);
  254. spin_lock(&kvm_lock);
  255. list_add(&kvm->vm_list, &vm_list);
  256. spin_unlock(&kvm_lock);
  257. kvm_io_bus_init(&kvm->mmio_bus);
  258. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  259. struct kvm_vcpu *vcpu = &kvm->vcpus[i];
  260. mutex_init(&vcpu->mutex);
  261. vcpu->cpu = -1;
  262. vcpu->kvm = kvm;
  263. vcpu->mmu.root_hpa = INVALID_PAGE;
  264. }
  265. return kvm;
  266. }
  267. static int kvm_dev_open(struct inode *inode, struct file *filp)
  268. {
  269. return 0;
  270. }
  271. /*
  272. * Free any memory in @free but not in @dont.
  273. */
  274. static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
  275. struct kvm_memory_slot *dont)
  276. {
  277. int i;
  278. if (!dont || free->phys_mem != dont->phys_mem)
  279. if (free->phys_mem) {
  280. for (i = 0; i < free->npages; ++i)
  281. if (free->phys_mem[i])
  282. __free_page(free->phys_mem[i]);
  283. vfree(free->phys_mem);
  284. }
  285. if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
  286. vfree(free->dirty_bitmap);
  287. free->phys_mem = NULL;
  288. free->npages = 0;
  289. free->dirty_bitmap = NULL;
  290. }
  291. static void kvm_free_physmem(struct kvm *kvm)
  292. {
  293. int i;
  294. for (i = 0; i < kvm->nmemslots; ++i)
  295. kvm_free_physmem_slot(&kvm->memslots[i], NULL);
  296. }
  297. static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
  298. {
  299. int i;
  300. for (i = 0; i < 2; ++i)
  301. if (vcpu->pio.guest_pages[i]) {
  302. __free_page(vcpu->pio.guest_pages[i]);
  303. vcpu->pio.guest_pages[i] = NULL;
  304. }
  305. }
  306. static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
  307. {
  308. if (!vcpu->vmcs)
  309. return;
  310. vcpu_load(vcpu);
  311. kvm_mmu_unload(vcpu);
  312. vcpu_put(vcpu);
  313. }
  314. static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
  315. {
  316. if (!vcpu->vmcs)
  317. return;
  318. vcpu_load(vcpu);
  319. kvm_mmu_destroy(vcpu);
  320. vcpu_put(vcpu);
  321. kvm_arch_ops->vcpu_free(vcpu);
  322. free_page((unsigned long)vcpu->run);
  323. vcpu->run = NULL;
  324. free_page((unsigned long)vcpu->pio_data);
  325. vcpu->pio_data = NULL;
  326. free_pio_guest_pages(vcpu);
  327. }
  328. static void kvm_free_vcpus(struct kvm *kvm)
  329. {
  330. unsigned int i;
  331. /*
  332. * Unpin any mmu pages first.
  333. */
  334. for (i = 0; i < KVM_MAX_VCPUS; ++i)
  335. kvm_unload_vcpu_mmu(&kvm->vcpus[i]);
  336. for (i = 0; i < KVM_MAX_VCPUS; ++i)
  337. kvm_free_vcpu(&kvm->vcpus[i]);
  338. }
  339. static int kvm_dev_release(struct inode *inode, struct file *filp)
  340. {
  341. return 0;
  342. }
  343. static void kvm_destroy_vm(struct kvm *kvm)
  344. {
  345. spin_lock(&kvm_lock);
  346. list_del(&kvm->vm_list);
  347. spin_unlock(&kvm_lock);
  348. kvm_io_bus_destroy(&kvm->pio_bus);
  349. kvm_io_bus_destroy(&kvm->mmio_bus);
  350. kvm_free_vcpus(kvm);
  351. kvm_free_physmem(kvm);
  352. kfree(kvm);
  353. }
  354. static int kvm_vm_release(struct inode *inode, struct file *filp)
  355. {
  356. struct kvm *kvm = filp->private_data;
  357. kvm_destroy_vm(kvm);
  358. return 0;
  359. }
  360. static void inject_gp(struct kvm_vcpu *vcpu)
  361. {
  362. kvm_arch_ops->inject_gp(vcpu, 0);
  363. }
  364. /*
  365. * Load the pae pdptrs. Return true is they are all valid.
  366. */
  367. static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
  368. {
  369. gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
  370. unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
  371. int i;
  372. u64 pdpte;
  373. u64 *pdpt;
  374. int ret;
  375. struct page *page;
  376. spin_lock(&vcpu->kvm->lock);
  377. page = gfn_to_page(vcpu->kvm, pdpt_gfn);
  378. /* FIXME: !page - emulate? 0xff? */
  379. pdpt = kmap_atomic(page, KM_USER0);
  380. ret = 1;
  381. for (i = 0; i < 4; ++i) {
  382. pdpte = pdpt[offset + i];
  383. if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
  384. ret = 0;
  385. goto out;
  386. }
  387. }
  388. for (i = 0; i < 4; ++i)
  389. vcpu->pdptrs[i] = pdpt[offset + i];
  390. out:
  391. kunmap_atomic(pdpt, KM_USER0);
  392. spin_unlock(&vcpu->kvm->lock);
  393. return ret;
  394. }
  395. void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  396. {
  397. if (cr0 & CR0_RESEVED_BITS) {
  398. printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
  399. cr0, vcpu->cr0);
  400. inject_gp(vcpu);
  401. return;
  402. }
  403. if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
  404. printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
  405. inject_gp(vcpu);
  406. return;
  407. }
  408. if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
  409. printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
  410. "and a clear PE flag\n");
  411. inject_gp(vcpu);
  412. return;
  413. }
  414. if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
  415. #ifdef CONFIG_X86_64
  416. if ((vcpu->shadow_efer & EFER_LME)) {
  417. int cs_db, cs_l;
  418. if (!is_pae(vcpu)) {
  419. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  420. "in long mode while PAE is disabled\n");
  421. inject_gp(vcpu);
  422. return;
  423. }
  424. kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  425. if (cs_l) {
  426. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  427. "in long mode while CS.L == 1\n");
  428. inject_gp(vcpu);
  429. return;
  430. }
  431. } else
  432. #endif
  433. if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
  434. printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
  435. "reserved bits\n");
  436. inject_gp(vcpu);
  437. return;
  438. }
  439. }
  440. kvm_arch_ops->set_cr0(vcpu, cr0);
  441. vcpu->cr0 = cr0;
  442. spin_lock(&vcpu->kvm->lock);
  443. kvm_mmu_reset_context(vcpu);
  444. spin_unlock(&vcpu->kvm->lock);
  445. return;
  446. }
  447. EXPORT_SYMBOL_GPL(set_cr0);
  448. void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
  449. {
  450. set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
  451. }
  452. EXPORT_SYMBOL_GPL(lmsw);
  453. void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  454. {
  455. if (cr4 & CR4_RESEVED_BITS) {
  456. printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
  457. inject_gp(vcpu);
  458. return;
  459. }
  460. if (is_long_mode(vcpu)) {
  461. if (!(cr4 & CR4_PAE_MASK)) {
  462. printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
  463. "in long mode\n");
  464. inject_gp(vcpu);
  465. return;
  466. }
  467. } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
  468. && !load_pdptrs(vcpu, vcpu->cr3)) {
  469. printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
  470. inject_gp(vcpu);
  471. }
  472. if (cr4 & CR4_VMXE_MASK) {
  473. printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
  474. inject_gp(vcpu);
  475. return;
  476. }
  477. kvm_arch_ops->set_cr4(vcpu, cr4);
  478. spin_lock(&vcpu->kvm->lock);
  479. kvm_mmu_reset_context(vcpu);
  480. spin_unlock(&vcpu->kvm->lock);
  481. }
  482. EXPORT_SYMBOL_GPL(set_cr4);
  483. void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
  484. {
  485. if (is_long_mode(vcpu)) {
  486. if (cr3 & CR3_L_MODE_RESEVED_BITS) {
  487. printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
  488. inject_gp(vcpu);
  489. return;
  490. }
  491. } else {
  492. if (cr3 & CR3_RESEVED_BITS) {
  493. printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
  494. inject_gp(vcpu);
  495. return;
  496. }
  497. if (is_paging(vcpu) && is_pae(vcpu) &&
  498. !load_pdptrs(vcpu, cr3)) {
  499. printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
  500. "reserved bits\n");
  501. inject_gp(vcpu);
  502. return;
  503. }
  504. }
  505. vcpu->cr3 = cr3;
  506. spin_lock(&vcpu->kvm->lock);
  507. /*
  508. * Does the new cr3 value map to physical memory? (Note, we
  509. * catch an invalid cr3 even in real-mode, because it would
  510. * cause trouble later on when we turn on paging anyway.)
  511. *
  512. * A real CPU would silently accept an invalid cr3 and would
  513. * attempt to use it - with largely undefined (and often hard
  514. * to debug) behavior on the guest side.
  515. */
  516. if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
  517. inject_gp(vcpu);
  518. else
  519. vcpu->mmu.new_cr3(vcpu);
  520. spin_unlock(&vcpu->kvm->lock);
  521. }
  522. EXPORT_SYMBOL_GPL(set_cr3);
  523. void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
  524. {
  525. if ( cr8 & CR8_RESEVED_BITS) {
  526. printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
  527. inject_gp(vcpu);
  528. return;
  529. }
  530. vcpu->cr8 = cr8;
  531. }
  532. EXPORT_SYMBOL_GPL(set_cr8);
  533. void fx_init(struct kvm_vcpu *vcpu)
  534. {
  535. struct __attribute__ ((__packed__)) fx_image_s {
  536. u16 control; //fcw
  537. u16 status; //fsw
  538. u16 tag; // ftw
  539. u16 opcode; //fop
  540. u64 ip; // fpu ip
  541. u64 operand;// fpu dp
  542. u32 mxcsr;
  543. u32 mxcsr_mask;
  544. } *fx_image;
  545. fx_save(vcpu->host_fx_image);
  546. fpu_init();
  547. fx_save(vcpu->guest_fx_image);
  548. fx_restore(vcpu->host_fx_image);
  549. fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
  550. fx_image->mxcsr = 0x1f80;
  551. memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
  552. 0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
  553. }
  554. EXPORT_SYMBOL_GPL(fx_init);
  555. /*
  556. * Allocate some memory and give it an address in the guest physical address
  557. * space.
  558. *
  559. * Discontiguous memory is allowed, mostly for framebuffers.
  560. */
  561. static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  562. struct kvm_memory_region *mem)
  563. {
  564. int r;
  565. gfn_t base_gfn;
  566. unsigned long npages;
  567. unsigned long i;
  568. struct kvm_memory_slot *memslot;
  569. struct kvm_memory_slot old, new;
  570. int memory_config_version;
  571. r = -EINVAL;
  572. /* General sanity checks */
  573. if (mem->memory_size & (PAGE_SIZE - 1))
  574. goto out;
  575. if (mem->guest_phys_addr & (PAGE_SIZE - 1))
  576. goto out;
  577. if (mem->slot >= KVM_MEMORY_SLOTS)
  578. goto out;
  579. if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
  580. goto out;
  581. memslot = &kvm->memslots[mem->slot];
  582. base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
  583. npages = mem->memory_size >> PAGE_SHIFT;
  584. if (!npages)
  585. mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
  586. raced:
  587. spin_lock(&kvm->lock);
  588. memory_config_version = kvm->memory_config_version;
  589. new = old = *memslot;
  590. new.base_gfn = base_gfn;
  591. new.npages = npages;
  592. new.flags = mem->flags;
  593. /* Disallow changing a memory slot's size. */
  594. r = -EINVAL;
  595. if (npages && old.npages && npages != old.npages)
  596. goto out_unlock;
  597. /* Check for overlaps */
  598. r = -EEXIST;
  599. for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
  600. struct kvm_memory_slot *s = &kvm->memslots[i];
  601. if (s == memslot)
  602. continue;
  603. if (!((base_gfn + npages <= s->base_gfn) ||
  604. (base_gfn >= s->base_gfn + s->npages)))
  605. goto out_unlock;
  606. }
  607. /*
  608. * Do memory allocations outside lock. memory_config_version will
  609. * detect any races.
  610. */
  611. spin_unlock(&kvm->lock);
  612. /* Deallocate if slot is being removed */
  613. if (!npages)
  614. new.phys_mem = NULL;
  615. /* Free page dirty bitmap if unneeded */
  616. if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
  617. new.dirty_bitmap = NULL;
  618. r = -ENOMEM;
  619. /* Allocate if a slot is being created */
  620. if (npages && !new.phys_mem) {
  621. new.phys_mem = vmalloc(npages * sizeof(struct page *));
  622. if (!new.phys_mem)
  623. goto out_free;
  624. memset(new.phys_mem, 0, npages * sizeof(struct page *));
  625. for (i = 0; i < npages; ++i) {
  626. new.phys_mem[i] = alloc_page(GFP_HIGHUSER
  627. | __GFP_ZERO);
  628. if (!new.phys_mem[i])
  629. goto out_free;
  630. set_page_private(new.phys_mem[i],0);
  631. }
  632. }
  633. /* Allocate page dirty bitmap if needed */
  634. if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
  635. unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
  636. new.dirty_bitmap = vmalloc(dirty_bytes);
  637. if (!new.dirty_bitmap)
  638. goto out_free;
  639. memset(new.dirty_bitmap, 0, dirty_bytes);
  640. }
  641. spin_lock(&kvm->lock);
  642. if (memory_config_version != kvm->memory_config_version) {
  643. spin_unlock(&kvm->lock);
  644. kvm_free_physmem_slot(&new, &old);
  645. goto raced;
  646. }
  647. r = -EAGAIN;
  648. if (kvm->busy)
  649. goto out_unlock;
  650. if (mem->slot >= kvm->nmemslots)
  651. kvm->nmemslots = mem->slot + 1;
  652. *memslot = new;
  653. ++kvm->memory_config_version;
  654. kvm_mmu_slot_remove_write_access(kvm, mem->slot);
  655. kvm_flush_remote_tlbs(kvm);
  656. spin_unlock(&kvm->lock);
  657. kvm_free_physmem_slot(&old, &new);
  658. return 0;
  659. out_unlock:
  660. spin_unlock(&kvm->lock);
  661. out_free:
  662. kvm_free_physmem_slot(&new, &old);
  663. out:
  664. return r;
  665. }
  666. /*
  667. * Get (and clear) the dirty memory log for a memory slot.
  668. */
  669. static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
  670. struct kvm_dirty_log *log)
  671. {
  672. struct kvm_memory_slot *memslot;
  673. int r, i;
  674. int n;
  675. unsigned long any = 0;
  676. spin_lock(&kvm->lock);
  677. /*
  678. * Prevent changes to guest memory configuration even while the lock
  679. * is not taken.
  680. */
  681. ++kvm->busy;
  682. spin_unlock(&kvm->lock);
  683. r = -EINVAL;
  684. if (log->slot >= KVM_MEMORY_SLOTS)
  685. goto out;
  686. memslot = &kvm->memslots[log->slot];
  687. r = -ENOENT;
  688. if (!memslot->dirty_bitmap)
  689. goto out;
  690. n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
  691. for (i = 0; !any && i < n/sizeof(long); ++i)
  692. any = memslot->dirty_bitmap[i];
  693. r = -EFAULT;
  694. if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
  695. goto out;
  696. spin_lock(&kvm->lock);
  697. kvm_mmu_slot_remove_write_access(kvm, log->slot);
  698. kvm_flush_remote_tlbs(kvm);
  699. memset(memslot->dirty_bitmap, 0, n);
  700. spin_unlock(&kvm->lock);
  701. r = 0;
  702. out:
  703. spin_lock(&kvm->lock);
  704. --kvm->busy;
  705. spin_unlock(&kvm->lock);
  706. return r;
  707. }
  708. /*
  709. * Set a new alias region. Aliases map a portion of physical memory into
  710. * another portion. This is useful for memory windows, for example the PC
  711. * VGA region.
  712. */
  713. static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
  714. struct kvm_memory_alias *alias)
  715. {
  716. int r, n;
  717. struct kvm_mem_alias *p;
  718. r = -EINVAL;
  719. /* General sanity checks */
  720. if (alias->memory_size & (PAGE_SIZE - 1))
  721. goto out;
  722. if (alias->guest_phys_addr & (PAGE_SIZE - 1))
  723. goto out;
  724. if (alias->slot >= KVM_ALIAS_SLOTS)
  725. goto out;
  726. if (alias->guest_phys_addr + alias->memory_size
  727. < alias->guest_phys_addr)
  728. goto out;
  729. if (alias->target_phys_addr + alias->memory_size
  730. < alias->target_phys_addr)
  731. goto out;
  732. spin_lock(&kvm->lock);
  733. p = &kvm->aliases[alias->slot];
  734. p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
  735. p->npages = alias->memory_size >> PAGE_SHIFT;
  736. p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
  737. for (n = KVM_ALIAS_SLOTS; n > 0; --n)
  738. if (kvm->aliases[n - 1].npages)
  739. break;
  740. kvm->naliases = n;
  741. kvm_mmu_zap_all(kvm);
  742. spin_unlock(&kvm->lock);
  743. return 0;
  744. out:
  745. return r;
  746. }
  747. static gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
  748. {
  749. int i;
  750. struct kvm_mem_alias *alias;
  751. for (i = 0; i < kvm->naliases; ++i) {
  752. alias = &kvm->aliases[i];
  753. if (gfn >= alias->base_gfn
  754. && gfn < alias->base_gfn + alias->npages)
  755. return alias->target_gfn + gfn - alias->base_gfn;
  756. }
  757. return gfn;
  758. }
  759. static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  760. {
  761. int i;
  762. for (i = 0; i < kvm->nmemslots; ++i) {
  763. struct kvm_memory_slot *memslot = &kvm->memslots[i];
  764. if (gfn >= memslot->base_gfn
  765. && gfn < memslot->base_gfn + memslot->npages)
  766. return memslot;
  767. }
  768. return NULL;
  769. }
  770. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  771. {
  772. gfn = unalias_gfn(kvm, gfn);
  773. return __gfn_to_memslot(kvm, gfn);
  774. }
  775. struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
  776. {
  777. struct kvm_memory_slot *slot;
  778. gfn = unalias_gfn(kvm, gfn);
  779. slot = __gfn_to_memslot(kvm, gfn);
  780. if (!slot)
  781. return NULL;
  782. return slot->phys_mem[gfn - slot->base_gfn];
  783. }
  784. EXPORT_SYMBOL_GPL(gfn_to_page);
  785. void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
  786. {
  787. int i;
  788. struct kvm_memory_slot *memslot;
  789. unsigned long rel_gfn;
  790. for (i = 0; i < kvm->nmemslots; ++i) {
  791. memslot = &kvm->memslots[i];
  792. if (gfn >= memslot->base_gfn
  793. && gfn < memslot->base_gfn + memslot->npages) {
  794. if (!memslot->dirty_bitmap)
  795. return;
  796. rel_gfn = gfn - memslot->base_gfn;
  797. /* avoid RMW */
  798. if (!test_bit(rel_gfn, memslot->dirty_bitmap))
  799. set_bit(rel_gfn, memslot->dirty_bitmap);
  800. return;
  801. }
  802. }
  803. }
  804. static int emulator_read_std(unsigned long addr,
  805. void *val,
  806. unsigned int bytes,
  807. struct x86_emulate_ctxt *ctxt)
  808. {
  809. struct kvm_vcpu *vcpu = ctxt->vcpu;
  810. void *data = val;
  811. while (bytes) {
  812. gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
  813. unsigned offset = addr & (PAGE_SIZE-1);
  814. unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
  815. unsigned long pfn;
  816. struct page *page;
  817. void *page_virt;
  818. if (gpa == UNMAPPED_GVA)
  819. return X86EMUL_PROPAGATE_FAULT;
  820. pfn = gpa >> PAGE_SHIFT;
  821. page = gfn_to_page(vcpu->kvm, pfn);
  822. if (!page)
  823. return X86EMUL_UNHANDLEABLE;
  824. page_virt = kmap_atomic(page, KM_USER0);
  825. memcpy(data, page_virt + offset, tocopy);
  826. kunmap_atomic(page_virt, KM_USER0);
  827. bytes -= tocopy;
  828. data += tocopy;
  829. addr += tocopy;
  830. }
  831. return X86EMUL_CONTINUE;
  832. }
  833. static int emulator_write_std(unsigned long addr,
  834. const void *val,
  835. unsigned int bytes,
  836. struct x86_emulate_ctxt *ctxt)
  837. {
  838. printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
  839. addr, bytes);
  840. return X86EMUL_UNHANDLEABLE;
  841. }
  842. static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
  843. gpa_t addr)
  844. {
  845. /*
  846. * Note that its important to have this wrapper function because
  847. * in the very near future we will be checking for MMIOs against
  848. * the LAPIC as well as the general MMIO bus
  849. */
  850. return kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr);
  851. }
  852. static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
  853. gpa_t addr)
  854. {
  855. return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr);
  856. }
  857. static int emulator_read_emulated(unsigned long addr,
  858. void *val,
  859. unsigned int bytes,
  860. struct x86_emulate_ctxt *ctxt)
  861. {
  862. struct kvm_vcpu *vcpu = ctxt->vcpu;
  863. struct kvm_io_device *mmio_dev;
  864. gpa_t gpa;
  865. if (vcpu->mmio_read_completed) {
  866. memcpy(val, vcpu->mmio_data, bytes);
  867. vcpu->mmio_read_completed = 0;
  868. return X86EMUL_CONTINUE;
  869. } else if (emulator_read_std(addr, val, bytes, ctxt)
  870. == X86EMUL_CONTINUE)
  871. return X86EMUL_CONTINUE;
  872. gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
  873. if (gpa == UNMAPPED_GVA)
  874. return X86EMUL_PROPAGATE_FAULT;
  875. /*
  876. * Is this MMIO handled locally?
  877. */
  878. mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
  879. if (mmio_dev) {
  880. kvm_iodevice_read(mmio_dev, gpa, bytes, val);
  881. return X86EMUL_CONTINUE;
  882. }
  883. vcpu->mmio_needed = 1;
  884. vcpu->mmio_phys_addr = gpa;
  885. vcpu->mmio_size = bytes;
  886. vcpu->mmio_is_write = 0;
  887. return X86EMUL_UNHANDLEABLE;
  888. }
  889. static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
  890. const void *val, int bytes)
  891. {
  892. struct page *page;
  893. void *virt;
  894. unsigned offset = offset_in_page(gpa);
  895. if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
  896. return 0;
  897. page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  898. if (!page)
  899. return 0;
  900. mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
  901. virt = kmap_atomic(page, KM_USER0);
  902. if (memcmp(virt + offset_in_page(gpa), val, bytes)) {
  903. kvm_mmu_pte_write(vcpu, gpa, virt + offset, val, bytes);
  904. memcpy(virt + offset_in_page(gpa), val, bytes);
  905. }
  906. kunmap_atomic(virt, KM_USER0);
  907. return 1;
  908. }
  909. static int emulator_write_emulated(unsigned long addr,
  910. const void *val,
  911. unsigned int bytes,
  912. struct x86_emulate_ctxt *ctxt)
  913. {
  914. struct kvm_vcpu *vcpu = ctxt->vcpu;
  915. struct kvm_io_device *mmio_dev;
  916. gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
  917. if (gpa == UNMAPPED_GVA) {
  918. kvm_arch_ops->inject_page_fault(vcpu, addr, 2);
  919. return X86EMUL_PROPAGATE_FAULT;
  920. }
  921. if (emulator_write_phys(vcpu, gpa, val, bytes))
  922. return X86EMUL_CONTINUE;
  923. /*
  924. * Is this MMIO handled locally?
  925. */
  926. mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
  927. if (mmio_dev) {
  928. kvm_iodevice_write(mmio_dev, gpa, bytes, val);
  929. return X86EMUL_CONTINUE;
  930. }
  931. vcpu->mmio_needed = 1;
  932. vcpu->mmio_phys_addr = gpa;
  933. vcpu->mmio_size = bytes;
  934. vcpu->mmio_is_write = 1;
  935. memcpy(vcpu->mmio_data, val, bytes);
  936. return X86EMUL_CONTINUE;
  937. }
  938. static int emulator_cmpxchg_emulated(unsigned long addr,
  939. const void *old,
  940. const void *new,
  941. unsigned int bytes,
  942. struct x86_emulate_ctxt *ctxt)
  943. {
  944. static int reported;
  945. if (!reported) {
  946. reported = 1;
  947. printk(KERN_WARNING "kvm: emulating exchange as write\n");
  948. }
  949. return emulator_write_emulated(addr, new, bytes, ctxt);
  950. }
  951. static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
  952. {
  953. return kvm_arch_ops->get_segment_base(vcpu, seg);
  954. }
  955. int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
  956. {
  957. return X86EMUL_CONTINUE;
  958. }
  959. int emulate_clts(struct kvm_vcpu *vcpu)
  960. {
  961. unsigned long cr0;
  962. cr0 = vcpu->cr0 & ~CR0_TS_MASK;
  963. kvm_arch_ops->set_cr0(vcpu, cr0);
  964. return X86EMUL_CONTINUE;
  965. }
  966. int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
  967. {
  968. struct kvm_vcpu *vcpu = ctxt->vcpu;
  969. switch (dr) {
  970. case 0 ... 3:
  971. *dest = kvm_arch_ops->get_dr(vcpu, dr);
  972. return X86EMUL_CONTINUE;
  973. default:
  974. printk(KERN_DEBUG "%s: unexpected dr %u\n",
  975. __FUNCTION__, dr);
  976. return X86EMUL_UNHANDLEABLE;
  977. }
  978. }
  979. int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
  980. {
  981. unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
  982. int exception;
  983. kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
  984. if (exception) {
  985. /* FIXME: better handling */
  986. return X86EMUL_UNHANDLEABLE;
  987. }
  988. return X86EMUL_CONTINUE;
  989. }
  990. static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
  991. {
  992. static int reported;
  993. u8 opcodes[4];
  994. unsigned long rip = ctxt->vcpu->rip;
  995. unsigned long rip_linear;
  996. rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
  997. if (reported)
  998. return;
  999. emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
  1000. printk(KERN_ERR "emulation failed but !mmio_needed?"
  1001. " rip %lx %02x %02x %02x %02x\n",
  1002. rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
  1003. reported = 1;
  1004. }
  1005. struct x86_emulate_ops emulate_ops = {
  1006. .read_std = emulator_read_std,
  1007. .write_std = emulator_write_std,
  1008. .read_emulated = emulator_read_emulated,
  1009. .write_emulated = emulator_write_emulated,
  1010. .cmpxchg_emulated = emulator_cmpxchg_emulated,
  1011. };
  1012. int emulate_instruction(struct kvm_vcpu *vcpu,
  1013. struct kvm_run *run,
  1014. unsigned long cr2,
  1015. u16 error_code)
  1016. {
  1017. struct x86_emulate_ctxt emulate_ctxt;
  1018. int r;
  1019. int cs_db, cs_l;
  1020. vcpu->mmio_fault_cr2 = cr2;
  1021. kvm_arch_ops->cache_regs(vcpu);
  1022. kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  1023. emulate_ctxt.vcpu = vcpu;
  1024. emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
  1025. emulate_ctxt.cr2 = cr2;
  1026. emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
  1027. ? X86EMUL_MODE_REAL : cs_l
  1028. ? X86EMUL_MODE_PROT64 : cs_db
  1029. ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
  1030. if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
  1031. emulate_ctxt.cs_base = 0;
  1032. emulate_ctxt.ds_base = 0;
  1033. emulate_ctxt.es_base = 0;
  1034. emulate_ctxt.ss_base = 0;
  1035. } else {
  1036. emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
  1037. emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
  1038. emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
  1039. emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
  1040. }
  1041. emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
  1042. emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
  1043. vcpu->mmio_is_write = 0;
  1044. r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
  1045. if ((r || vcpu->mmio_is_write) && run) {
  1046. run->mmio.phys_addr = vcpu->mmio_phys_addr;
  1047. memcpy(run->mmio.data, vcpu->mmio_data, 8);
  1048. run->mmio.len = vcpu->mmio_size;
  1049. run->mmio.is_write = vcpu->mmio_is_write;
  1050. }
  1051. if (r) {
  1052. if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
  1053. return EMULATE_DONE;
  1054. if (!vcpu->mmio_needed) {
  1055. report_emulation_failure(&emulate_ctxt);
  1056. return EMULATE_FAIL;
  1057. }
  1058. return EMULATE_DO_MMIO;
  1059. }
  1060. kvm_arch_ops->decache_regs(vcpu);
  1061. kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
  1062. if (vcpu->mmio_is_write) {
  1063. vcpu->mmio_needed = 0;
  1064. return EMULATE_DO_MMIO;
  1065. }
  1066. return EMULATE_DONE;
  1067. }
  1068. EXPORT_SYMBOL_GPL(emulate_instruction);
  1069. int kvm_emulate_halt(struct kvm_vcpu *vcpu)
  1070. {
  1071. if (vcpu->irq_summary)
  1072. return 1;
  1073. vcpu->run->exit_reason = KVM_EXIT_HLT;
  1074. ++vcpu->stat.halt_exits;
  1075. return 0;
  1076. }
  1077. EXPORT_SYMBOL_GPL(kvm_emulate_halt);
  1078. int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
  1079. {
  1080. unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
  1081. kvm_arch_ops->cache_regs(vcpu);
  1082. ret = -KVM_EINVAL;
  1083. #ifdef CONFIG_X86_64
  1084. if (is_long_mode(vcpu)) {
  1085. nr = vcpu->regs[VCPU_REGS_RAX];
  1086. a0 = vcpu->regs[VCPU_REGS_RDI];
  1087. a1 = vcpu->regs[VCPU_REGS_RSI];
  1088. a2 = vcpu->regs[VCPU_REGS_RDX];
  1089. a3 = vcpu->regs[VCPU_REGS_RCX];
  1090. a4 = vcpu->regs[VCPU_REGS_R8];
  1091. a5 = vcpu->regs[VCPU_REGS_R9];
  1092. } else
  1093. #endif
  1094. {
  1095. nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
  1096. a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
  1097. a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
  1098. a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
  1099. a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
  1100. a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
  1101. a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
  1102. }
  1103. switch (nr) {
  1104. default:
  1105. run->hypercall.args[0] = a0;
  1106. run->hypercall.args[1] = a1;
  1107. run->hypercall.args[2] = a2;
  1108. run->hypercall.args[3] = a3;
  1109. run->hypercall.args[4] = a4;
  1110. run->hypercall.args[5] = a5;
  1111. run->hypercall.ret = ret;
  1112. run->hypercall.longmode = is_long_mode(vcpu);
  1113. kvm_arch_ops->decache_regs(vcpu);
  1114. return 0;
  1115. }
  1116. vcpu->regs[VCPU_REGS_RAX] = ret;
  1117. kvm_arch_ops->decache_regs(vcpu);
  1118. return 1;
  1119. }
  1120. EXPORT_SYMBOL_GPL(kvm_hypercall);
  1121. static u64 mk_cr_64(u64 curr_cr, u32 new_val)
  1122. {
  1123. return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
  1124. }
  1125. void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  1126. {
  1127. struct descriptor_table dt = { limit, base };
  1128. kvm_arch_ops->set_gdt(vcpu, &dt);
  1129. }
  1130. void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  1131. {
  1132. struct descriptor_table dt = { limit, base };
  1133. kvm_arch_ops->set_idt(vcpu, &dt);
  1134. }
  1135. void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
  1136. unsigned long *rflags)
  1137. {
  1138. lmsw(vcpu, msw);
  1139. *rflags = kvm_arch_ops->get_rflags(vcpu);
  1140. }
  1141. unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
  1142. {
  1143. kvm_arch_ops->decache_cr4_guest_bits(vcpu);
  1144. switch (cr) {
  1145. case 0:
  1146. return vcpu->cr0;
  1147. case 2:
  1148. return vcpu->cr2;
  1149. case 3:
  1150. return vcpu->cr3;
  1151. case 4:
  1152. return vcpu->cr4;
  1153. default:
  1154. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
  1155. return 0;
  1156. }
  1157. }
  1158. void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
  1159. unsigned long *rflags)
  1160. {
  1161. switch (cr) {
  1162. case 0:
  1163. set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
  1164. *rflags = kvm_arch_ops->get_rflags(vcpu);
  1165. break;
  1166. case 2:
  1167. vcpu->cr2 = val;
  1168. break;
  1169. case 3:
  1170. set_cr3(vcpu, val);
  1171. break;
  1172. case 4:
  1173. set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
  1174. break;
  1175. default:
  1176. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
  1177. }
  1178. }
  1179. /*
  1180. * Register the para guest with the host:
  1181. */
  1182. static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
  1183. {
  1184. struct kvm_vcpu_para_state *para_state;
  1185. hpa_t para_state_hpa, hypercall_hpa;
  1186. struct page *para_state_page;
  1187. unsigned char *hypercall;
  1188. gpa_t hypercall_gpa;
  1189. printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
  1190. printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
  1191. /*
  1192. * Needs to be page aligned:
  1193. */
  1194. if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
  1195. goto err_gp;
  1196. para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
  1197. printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
  1198. if (is_error_hpa(para_state_hpa))
  1199. goto err_gp;
  1200. mark_page_dirty(vcpu->kvm, para_state_gpa >> PAGE_SHIFT);
  1201. para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
  1202. para_state = kmap_atomic(para_state_page, KM_USER0);
  1203. printk(KERN_DEBUG ".... guest version: %d\n", para_state->guest_version);
  1204. printk(KERN_DEBUG ".... size: %d\n", para_state->size);
  1205. para_state->host_version = KVM_PARA_API_VERSION;
  1206. /*
  1207. * We cannot support guests that try to register themselves
  1208. * with a newer API version than the host supports:
  1209. */
  1210. if (para_state->guest_version > KVM_PARA_API_VERSION) {
  1211. para_state->ret = -KVM_EINVAL;
  1212. goto err_kunmap_skip;
  1213. }
  1214. hypercall_gpa = para_state->hypercall_gpa;
  1215. hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
  1216. printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
  1217. if (is_error_hpa(hypercall_hpa)) {
  1218. para_state->ret = -KVM_EINVAL;
  1219. goto err_kunmap_skip;
  1220. }
  1221. printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
  1222. vcpu->para_state_page = para_state_page;
  1223. vcpu->para_state_gpa = para_state_gpa;
  1224. vcpu->hypercall_gpa = hypercall_gpa;
  1225. mark_page_dirty(vcpu->kvm, hypercall_gpa >> PAGE_SHIFT);
  1226. hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
  1227. KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
  1228. kvm_arch_ops->patch_hypercall(vcpu, hypercall);
  1229. kunmap_atomic(hypercall, KM_USER1);
  1230. para_state->ret = 0;
  1231. err_kunmap_skip:
  1232. kunmap_atomic(para_state, KM_USER0);
  1233. return 0;
  1234. err_gp:
  1235. return 1;
  1236. }
  1237. int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1238. {
  1239. u64 data;
  1240. switch (msr) {
  1241. case 0xc0010010: /* SYSCFG */
  1242. case 0xc0010015: /* HWCR */
  1243. case MSR_IA32_PLATFORM_ID:
  1244. case MSR_IA32_P5_MC_ADDR:
  1245. case MSR_IA32_P5_MC_TYPE:
  1246. case MSR_IA32_MC0_CTL:
  1247. case MSR_IA32_MCG_STATUS:
  1248. case MSR_IA32_MCG_CAP:
  1249. case MSR_IA32_MC0_MISC:
  1250. case MSR_IA32_MC0_MISC+4:
  1251. case MSR_IA32_MC0_MISC+8:
  1252. case MSR_IA32_MC0_MISC+12:
  1253. case MSR_IA32_MC0_MISC+16:
  1254. case MSR_IA32_UCODE_REV:
  1255. case MSR_IA32_PERF_STATUS:
  1256. case MSR_IA32_EBL_CR_POWERON:
  1257. /* MTRR registers */
  1258. case 0xfe:
  1259. case 0x200 ... 0x2ff:
  1260. data = 0;
  1261. break;
  1262. case 0xcd: /* fsb frequency */
  1263. data = 3;
  1264. break;
  1265. case MSR_IA32_APICBASE:
  1266. data = vcpu->apic_base;
  1267. break;
  1268. case MSR_IA32_MISC_ENABLE:
  1269. data = vcpu->ia32_misc_enable_msr;
  1270. break;
  1271. #ifdef CONFIG_X86_64
  1272. case MSR_EFER:
  1273. data = vcpu->shadow_efer;
  1274. break;
  1275. #endif
  1276. default:
  1277. printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
  1278. return 1;
  1279. }
  1280. *pdata = data;
  1281. return 0;
  1282. }
  1283. EXPORT_SYMBOL_GPL(kvm_get_msr_common);
  1284. /*
  1285. * Reads an msr value (of 'msr_index') into 'pdata'.
  1286. * Returns 0 on success, non-0 otherwise.
  1287. * Assumes vcpu_load() was already called.
  1288. */
  1289. int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
  1290. {
  1291. return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
  1292. }
  1293. #ifdef CONFIG_X86_64
  1294. static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
  1295. {
  1296. if (efer & EFER_RESERVED_BITS) {
  1297. printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
  1298. efer);
  1299. inject_gp(vcpu);
  1300. return;
  1301. }
  1302. if (is_paging(vcpu)
  1303. && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
  1304. printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
  1305. inject_gp(vcpu);
  1306. return;
  1307. }
  1308. kvm_arch_ops->set_efer(vcpu, efer);
  1309. efer &= ~EFER_LMA;
  1310. efer |= vcpu->shadow_efer & EFER_LMA;
  1311. vcpu->shadow_efer = efer;
  1312. }
  1313. #endif
  1314. int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1315. {
  1316. switch (msr) {
  1317. #ifdef CONFIG_X86_64
  1318. case MSR_EFER:
  1319. set_efer(vcpu, data);
  1320. break;
  1321. #endif
  1322. case MSR_IA32_MC0_STATUS:
  1323. printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
  1324. __FUNCTION__, data);
  1325. break;
  1326. case MSR_IA32_MCG_STATUS:
  1327. printk(KERN_WARNING "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
  1328. __FUNCTION__, data);
  1329. break;
  1330. case MSR_IA32_UCODE_REV:
  1331. case MSR_IA32_UCODE_WRITE:
  1332. case 0x200 ... 0x2ff: /* MTRRs */
  1333. break;
  1334. case MSR_IA32_APICBASE:
  1335. vcpu->apic_base = data;
  1336. break;
  1337. case MSR_IA32_MISC_ENABLE:
  1338. vcpu->ia32_misc_enable_msr = data;
  1339. break;
  1340. /*
  1341. * This is the 'probe whether the host is KVM' logic:
  1342. */
  1343. case MSR_KVM_API_MAGIC:
  1344. return vcpu_register_para(vcpu, data);
  1345. default:
  1346. printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
  1347. return 1;
  1348. }
  1349. return 0;
  1350. }
  1351. EXPORT_SYMBOL_GPL(kvm_set_msr_common);
  1352. /*
  1353. * Writes msr value into into the appropriate "register".
  1354. * Returns 0 on success, non-0 otherwise.
  1355. * Assumes vcpu_load() was already called.
  1356. */
  1357. int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
  1358. {
  1359. return kvm_arch_ops->set_msr(vcpu, msr_index, data);
  1360. }
  1361. void kvm_resched(struct kvm_vcpu *vcpu)
  1362. {
  1363. if (!need_resched())
  1364. return;
  1365. vcpu_put(vcpu);
  1366. cond_resched();
  1367. vcpu_load(vcpu);
  1368. }
  1369. EXPORT_SYMBOL_GPL(kvm_resched);
  1370. void load_msrs(struct vmx_msr_entry *e, int n)
  1371. {
  1372. int i;
  1373. for (i = 0; i < n; ++i)
  1374. wrmsrl(e[i].index, e[i].data);
  1375. }
  1376. EXPORT_SYMBOL_GPL(load_msrs);
  1377. void save_msrs(struct vmx_msr_entry *e, int n)
  1378. {
  1379. int i;
  1380. for (i = 0; i < n; ++i)
  1381. rdmsrl(e[i].index, e[i].data);
  1382. }
  1383. EXPORT_SYMBOL_GPL(save_msrs);
  1384. void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
  1385. {
  1386. int i;
  1387. u32 function;
  1388. struct kvm_cpuid_entry *e, *best;
  1389. kvm_arch_ops->cache_regs(vcpu);
  1390. function = vcpu->regs[VCPU_REGS_RAX];
  1391. vcpu->regs[VCPU_REGS_RAX] = 0;
  1392. vcpu->regs[VCPU_REGS_RBX] = 0;
  1393. vcpu->regs[VCPU_REGS_RCX] = 0;
  1394. vcpu->regs[VCPU_REGS_RDX] = 0;
  1395. best = NULL;
  1396. for (i = 0; i < vcpu->cpuid_nent; ++i) {
  1397. e = &vcpu->cpuid_entries[i];
  1398. if (e->function == function) {
  1399. best = e;
  1400. break;
  1401. }
  1402. /*
  1403. * Both basic or both extended?
  1404. */
  1405. if (((e->function ^ function) & 0x80000000) == 0)
  1406. if (!best || e->function > best->function)
  1407. best = e;
  1408. }
  1409. if (best) {
  1410. vcpu->regs[VCPU_REGS_RAX] = best->eax;
  1411. vcpu->regs[VCPU_REGS_RBX] = best->ebx;
  1412. vcpu->regs[VCPU_REGS_RCX] = best->ecx;
  1413. vcpu->regs[VCPU_REGS_RDX] = best->edx;
  1414. }
  1415. kvm_arch_ops->decache_regs(vcpu);
  1416. kvm_arch_ops->skip_emulated_instruction(vcpu);
  1417. }
  1418. EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
  1419. static int pio_copy_data(struct kvm_vcpu *vcpu)
  1420. {
  1421. void *p = vcpu->pio_data;
  1422. void *q;
  1423. unsigned bytes;
  1424. int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1;
  1425. kvm_arch_ops->vcpu_put(vcpu);
  1426. q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
  1427. PAGE_KERNEL);
  1428. if (!q) {
  1429. kvm_arch_ops->vcpu_load(vcpu);
  1430. free_pio_guest_pages(vcpu);
  1431. return -ENOMEM;
  1432. }
  1433. q += vcpu->pio.guest_page_offset;
  1434. bytes = vcpu->pio.size * vcpu->pio.cur_count;
  1435. if (vcpu->pio.in)
  1436. memcpy(q, p, bytes);
  1437. else
  1438. memcpy(p, q, bytes);
  1439. q -= vcpu->pio.guest_page_offset;
  1440. vunmap(q);
  1441. kvm_arch_ops->vcpu_load(vcpu);
  1442. free_pio_guest_pages(vcpu);
  1443. return 0;
  1444. }
  1445. static int complete_pio(struct kvm_vcpu *vcpu)
  1446. {
  1447. struct kvm_pio_request *io = &vcpu->pio;
  1448. long delta;
  1449. int r;
  1450. kvm_arch_ops->cache_regs(vcpu);
  1451. if (!io->string) {
  1452. if (io->in)
  1453. memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data,
  1454. io->size);
  1455. } else {
  1456. if (io->in) {
  1457. r = pio_copy_data(vcpu);
  1458. if (r) {
  1459. kvm_arch_ops->cache_regs(vcpu);
  1460. return r;
  1461. }
  1462. }
  1463. delta = 1;
  1464. if (io->rep) {
  1465. delta *= io->cur_count;
  1466. /*
  1467. * The size of the register should really depend on
  1468. * current address size.
  1469. */
  1470. vcpu->regs[VCPU_REGS_RCX] -= delta;
  1471. }
  1472. if (io->down)
  1473. delta = -delta;
  1474. delta *= io->size;
  1475. if (io->in)
  1476. vcpu->regs[VCPU_REGS_RDI] += delta;
  1477. else
  1478. vcpu->regs[VCPU_REGS_RSI] += delta;
  1479. }
  1480. kvm_arch_ops->decache_regs(vcpu);
  1481. io->count -= io->cur_count;
  1482. io->cur_count = 0;
  1483. if (!io->count)
  1484. kvm_arch_ops->skip_emulated_instruction(vcpu);
  1485. return 0;
  1486. }
  1487. void kernel_pio(struct kvm_io_device *pio_dev, struct kvm_vcpu *vcpu)
  1488. {
  1489. /* TODO: String I/O for in kernel device */
  1490. if (vcpu->pio.in)
  1491. kvm_iodevice_read(pio_dev, vcpu->pio.port,
  1492. vcpu->pio.size,
  1493. vcpu->pio_data);
  1494. else
  1495. kvm_iodevice_write(pio_dev, vcpu->pio.port,
  1496. vcpu->pio.size,
  1497. vcpu->pio_data);
  1498. }
  1499. int kvm_setup_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
  1500. int size, unsigned long count, int string, int down,
  1501. gva_t address, int rep, unsigned port)
  1502. {
  1503. unsigned now, in_page;
  1504. int i;
  1505. int nr_pages = 1;
  1506. struct page *page;
  1507. struct kvm_io_device *pio_dev;
  1508. vcpu->run->exit_reason = KVM_EXIT_IO;
  1509. vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  1510. vcpu->run->io.size = size;
  1511. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  1512. vcpu->run->io.count = count;
  1513. vcpu->run->io.port = port;
  1514. vcpu->pio.count = count;
  1515. vcpu->pio.cur_count = count;
  1516. vcpu->pio.size = size;
  1517. vcpu->pio.in = in;
  1518. vcpu->pio.port = port;
  1519. vcpu->pio.string = string;
  1520. vcpu->pio.down = down;
  1521. vcpu->pio.guest_page_offset = offset_in_page(address);
  1522. vcpu->pio.rep = rep;
  1523. pio_dev = vcpu_find_pio_dev(vcpu, port);
  1524. if (!string) {
  1525. kvm_arch_ops->cache_regs(vcpu);
  1526. memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4);
  1527. kvm_arch_ops->decache_regs(vcpu);
  1528. if (pio_dev) {
  1529. kernel_pio(pio_dev, vcpu);
  1530. complete_pio(vcpu);
  1531. return 1;
  1532. }
  1533. return 0;
  1534. }
  1535. /* TODO: String I/O for in kernel device */
  1536. if (pio_dev)
  1537. printk(KERN_ERR "kvm_setup_pio: no string io support\n");
  1538. if (!count) {
  1539. kvm_arch_ops->skip_emulated_instruction(vcpu);
  1540. return 1;
  1541. }
  1542. now = min(count, PAGE_SIZE / size);
  1543. if (!down)
  1544. in_page = PAGE_SIZE - offset_in_page(address);
  1545. else
  1546. in_page = offset_in_page(address) + size;
  1547. now = min(count, (unsigned long)in_page / size);
  1548. if (!now) {
  1549. /*
  1550. * String I/O straddles page boundary. Pin two guest pages
  1551. * so that we satisfy atomicity constraints. Do just one
  1552. * transaction to avoid complexity.
  1553. */
  1554. nr_pages = 2;
  1555. now = 1;
  1556. }
  1557. if (down) {
  1558. /*
  1559. * String I/O in reverse. Yuck. Kill the guest, fix later.
  1560. */
  1561. printk(KERN_ERR "kvm: guest string pio down\n");
  1562. inject_gp(vcpu);
  1563. return 1;
  1564. }
  1565. vcpu->run->io.count = now;
  1566. vcpu->pio.cur_count = now;
  1567. for (i = 0; i < nr_pages; ++i) {
  1568. spin_lock(&vcpu->kvm->lock);
  1569. page = gva_to_page(vcpu, address + i * PAGE_SIZE);
  1570. if (page)
  1571. get_page(page);
  1572. vcpu->pio.guest_pages[i] = page;
  1573. spin_unlock(&vcpu->kvm->lock);
  1574. if (!page) {
  1575. inject_gp(vcpu);
  1576. free_pio_guest_pages(vcpu);
  1577. return 1;
  1578. }
  1579. }
  1580. if (!vcpu->pio.in)
  1581. return pio_copy_data(vcpu);
  1582. return 0;
  1583. }
  1584. EXPORT_SYMBOL_GPL(kvm_setup_pio);
  1585. static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1586. {
  1587. int r;
  1588. sigset_t sigsaved;
  1589. vcpu_load(vcpu);
  1590. if (vcpu->sigset_active)
  1591. sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
  1592. /* re-sync apic's tpr */
  1593. vcpu->cr8 = kvm_run->cr8;
  1594. if (vcpu->pio.cur_count) {
  1595. r = complete_pio(vcpu);
  1596. if (r)
  1597. goto out;
  1598. }
  1599. if (vcpu->mmio_needed) {
  1600. memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
  1601. vcpu->mmio_read_completed = 1;
  1602. vcpu->mmio_needed = 0;
  1603. r = emulate_instruction(vcpu, kvm_run,
  1604. vcpu->mmio_fault_cr2, 0);
  1605. if (r == EMULATE_DO_MMIO) {
  1606. /*
  1607. * Read-modify-write. Back to userspace.
  1608. */
  1609. kvm_run->exit_reason = KVM_EXIT_MMIO;
  1610. r = 0;
  1611. goto out;
  1612. }
  1613. }
  1614. if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
  1615. kvm_arch_ops->cache_regs(vcpu);
  1616. vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
  1617. kvm_arch_ops->decache_regs(vcpu);
  1618. }
  1619. r = kvm_arch_ops->run(vcpu, kvm_run);
  1620. out:
  1621. if (vcpu->sigset_active)
  1622. sigprocmask(SIG_SETMASK, &sigsaved, NULL);
  1623. vcpu_put(vcpu);
  1624. return r;
  1625. }
  1626. static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
  1627. struct kvm_regs *regs)
  1628. {
  1629. vcpu_load(vcpu);
  1630. kvm_arch_ops->cache_regs(vcpu);
  1631. regs->rax = vcpu->regs[VCPU_REGS_RAX];
  1632. regs->rbx = vcpu->regs[VCPU_REGS_RBX];
  1633. regs->rcx = vcpu->regs[VCPU_REGS_RCX];
  1634. regs->rdx = vcpu->regs[VCPU_REGS_RDX];
  1635. regs->rsi = vcpu->regs[VCPU_REGS_RSI];
  1636. regs->rdi = vcpu->regs[VCPU_REGS_RDI];
  1637. regs->rsp = vcpu->regs[VCPU_REGS_RSP];
  1638. regs->rbp = vcpu->regs[VCPU_REGS_RBP];
  1639. #ifdef CONFIG_X86_64
  1640. regs->r8 = vcpu->regs[VCPU_REGS_R8];
  1641. regs->r9 = vcpu->regs[VCPU_REGS_R9];
  1642. regs->r10 = vcpu->regs[VCPU_REGS_R10];
  1643. regs->r11 = vcpu->regs[VCPU_REGS_R11];
  1644. regs->r12 = vcpu->regs[VCPU_REGS_R12];
  1645. regs->r13 = vcpu->regs[VCPU_REGS_R13];
  1646. regs->r14 = vcpu->regs[VCPU_REGS_R14];
  1647. regs->r15 = vcpu->regs[VCPU_REGS_R15];
  1648. #endif
  1649. regs->rip = vcpu->rip;
  1650. regs->rflags = kvm_arch_ops->get_rflags(vcpu);
  1651. /*
  1652. * Don't leak debug flags in case they were set for guest debugging
  1653. */
  1654. if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
  1655. regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
  1656. vcpu_put(vcpu);
  1657. return 0;
  1658. }
  1659. static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
  1660. struct kvm_regs *regs)
  1661. {
  1662. vcpu_load(vcpu);
  1663. vcpu->regs[VCPU_REGS_RAX] = regs->rax;
  1664. vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
  1665. vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
  1666. vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
  1667. vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
  1668. vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
  1669. vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
  1670. vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
  1671. #ifdef CONFIG_X86_64
  1672. vcpu->regs[VCPU_REGS_R8] = regs->r8;
  1673. vcpu->regs[VCPU_REGS_R9] = regs->r9;
  1674. vcpu->regs[VCPU_REGS_R10] = regs->r10;
  1675. vcpu->regs[VCPU_REGS_R11] = regs->r11;
  1676. vcpu->regs[VCPU_REGS_R12] = regs->r12;
  1677. vcpu->regs[VCPU_REGS_R13] = regs->r13;
  1678. vcpu->regs[VCPU_REGS_R14] = regs->r14;
  1679. vcpu->regs[VCPU_REGS_R15] = regs->r15;
  1680. #endif
  1681. vcpu->rip = regs->rip;
  1682. kvm_arch_ops->set_rflags(vcpu, regs->rflags);
  1683. kvm_arch_ops->decache_regs(vcpu);
  1684. vcpu_put(vcpu);
  1685. return 0;
  1686. }
  1687. static void get_segment(struct kvm_vcpu *vcpu,
  1688. struct kvm_segment *var, int seg)
  1689. {
  1690. return kvm_arch_ops->get_segment(vcpu, var, seg);
  1691. }
  1692. static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  1693. struct kvm_sregs *sregs)
  1694. {
  1695. struct descriptor_table dt;
  1696. vcpu_load(vcpu);
  1697. get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  1698. get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  1699. get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  1700. get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  1701. get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  1702. get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  1703. get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  1704. get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  1705. kvm_arch_ops->get_idt(vcpu, &dt);
  1706. sregs->idt.limit = dt.limit;
  1707. sregs->idt.base = dt.base;
  1708. kvm_arch_ops->get_gdt(vcpu, &dt);
  1709. sregs->gdt.limit = dt.limit;
  1710. sregs->gdt.base = dt.base;
  1711. kvm_arch_ops->decache_cr4_guest_bits(vcpu);
  1712. sregs->cr0 = vcpu->cr0;
  1713. sregs->cr2 = vcpu->cr2;
  1714. sregs->cr3 = vcpu->cr3;
  1715. sregs->cr4 = vcpu->cr4;
  1716. sregs->cr8 = vcpu->cr8;
  1717. sregs->efer = vcpu->shadow_efer;
  1718. sregs->apic_base = vcpu->apic_base;
  1719. memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
  1720. sizeof sregs->interrupt_bitmap);
  1721. vcpu_put(vcpu);
  1722. return 0;
  1723. }
  1724. static void set_segment(struct kvm_vcpu *vcpu,
  1725. struct kvm_segment *var, int seg)
  1726. {
  1727. return kvm_arch_ops->set_segment(vcpu, var, seg);
  1728. }
  1729. static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  1730. struct kvm_sregs *sregs)
  1731. {
  1732. int mmu_reset_needed = 0;
  1733. int i;
  1734. struct descriptor_table dt;
  1735. vcpu_load(vcpu);
  1736. dt.limit = sregs->idt.limit;
  1737. dt.base = sregs->idt.base;
  1738. kvm_arch_ops->set_idt(vcpu, &dt);
  1739. dt.limit = sregs->gdt.limit;
  1740. dt.base = sregs->gdt.base;
  1741. kvm_arch_ops->set_gdt(vcpu, &dt);
  1742. vcpu->cr2 = sregs->cr2;
  1743. mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
  1744. vcpu->cr3 = sregs->cr3;
  1745. vcpu->cr8 = sregs->cr8;
  1746. mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
  1747. #ifdef CONFIG_X86_64
  1748. kvm_arch_ops->set_efer(vcpu, sregs->efer);
  1749. #endif
  1750. vcpu->apic_base = sregs->apic_base;
  1751. kvm_arch_ops->decache_cr4_guest_bits(vcpu);
  1752. mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
  1753. kvm_arch_ops->set_cr0(vcpu, sregs->cr0);
  1754. mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
  1755. kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
  1756. if (!is_long_mode(vcpu) && is_pae(vcpu))
  1757. load_pdptrs(vcpu, vcpu->cr3);
  1758. if (mmu_reset_needed)
  1759. kvm_mmu_reset_context(vcpu);
  1760. memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
  1761. sizeof vcpu->irq_pending);
  1762. vcpu->irq_summary = 0;
  1763. for (i = 0; i < NR_IRQ_WORDS; ++i)
  1764. if (vcpu->irq_pending[i])
  1765. __set_bit(i, &vcpu->irq_summary);
  1766. set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  1767. set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  1768. set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  1769. set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  1770. set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  1771. set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  1772. set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  1773. set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  1774. vcpu_put(vcpu);
  1775. return 0;
  1776. }
  1777. /*
  1778. * List of msr numbers which we expose to userspace through KVM_GET_MSRS
  1779. * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
  1780. *
  1781. * This list is modified at module load time to reflect the
  1782. * capabilities of the host cpu.
  1783. */
  1784. static u32 msrs_to_save[] = {
  1785. MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  1786. MSR_K6_STAR,
  1787. #ifdef CONFIG_X86_64
  1788. MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
  1789. #endif
  1790. MSR_IA32_TIME_STAMP_COUNTER,
  1791. };
  1792. static unsigned num_msrs_to_save;
  1793. static u32 emulated_msrs[] = {
  1794. MSR_IA32_MISC_ENABLE,
  1795. };
  1796. static __init void kvm_init_msr_list(void)
  1797. {
  1798. u32 dummy[2];
  1799. unsigned i, j;
  1800. for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
  1801. if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
  1802. continue;
  1803. if (j < i)
  1804. msrs_to_save[j] = msrs_to_save[i];
  1805. j++;
  1806. }
  1807. num_msrs_to_save = j;
  1808. }
  1809. /*
  1810. * Adapt set_msr() to msr_io()'s calling convention
  1811. */
  1812. static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
  1813. {
  1814. return kvm_set_msr(vcpu, index, *data);
  1815. }
  1816. /*
  1817. * Read or write a bunch of msrs. All parameters are kernel addresses.
  1818. *
  1819. * @return number of msrs set successfully.
  1820. */
  1821. static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
  1822. struct kvm_msr_entry *entries,
  1823. int (*do_msr)(struct kvm_vcpu *vcpu,
  1824. unsigned index, u64 *data))
  1825. {
  1826. int i;
  1827. vcpu_load(vcpu);
  1828. for (i = 0; i < msrs->nmsrs; ++i)
  1829. if (do_msr(vcpu, entries[i].index, &entries[i].data))
  1830. break;
  1831. vcpu_put(vcpu);
  1832. return i;
  1833. }
  1834. /*
  1835. * Read or write a bunch of msrs. Parameters are user addresses.
  1836. *
  1837. * @return number of msrs set successfully.
  1838. */
  1839. static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
  1840. int (*do_msr)(struct kvm_vcpu *vcpu,
  1841. unsigned index, u64 *data),
  1842. int writeback)
  1843. {
  1844. struct kvm_msrs msrs;
  1845. struct kvm_msr_entry *entries;
  1846. int r, n;
  1847. unsigned size;
  1848. r = -EFAULT;
  1849. if (copy_from_user(&msrs, user_msrs, sizeof msrs))
  1850. goto out;
  1851. r = -E2BIG;
  1852. if (msrs.nmsrs >= MAX_IO_MSRS)
  1853. goto out;
  1854. r = -ENOMEM;
  1855. size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
  1856. entries = vmalloc(size);
  1857. if (!entries)
  1858. goto out;
  1859. r = -EFAULT;
  1860. if (copy_from_user(entries, user_msrs->entries, size))
  1861. goto out_free;
  1862. r = n = __msr_io(vcpu, &msrs, entries, do_msr);
  1863. if (r < 0)
  1864. goto out_free;
  1865. r = -EFAULT;
  1866. if (writeback && copy_to_user(user_msrs->entries, entries, size))
  1867. goto out_free;
  1868. r = n;
  1869. out_free:
  1870. vfree(entries);
  1871. out:
  1872. return r;
  1873. }
  1874. /*
  1875. * Translate a guest virtual address to a guest physical address.
  1876. */
  1877. static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  1878. struct kvm_translation *tr)
  1879. {
  1880. unsigned long vaddr = tr->linear_address;
  1881. gpa_t gpa;
  1882. vcpu_load(vcpu);
  1883. spin_lock(&vcpu->kvm->lock);
  1884. gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
  1885. tr->physical_address = gpa;
  1886. tr->valid = gpa != UNMAPPED_GVA;
  1887. tr->writeable = 1;
  1888. tr->usermode = 0;
  1889. spin_unlock(&vcpu->kvm->lock);
  1890. vcpu_put(vcpu);
  1891. return 0;
  1892. }
  1893. static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
  1894. struct kvm_interrupt *irq)
  1895. {
  1896. if (irq->irq < 0 || irq->irq >= 256)
  1897. return -EINVAL;
  1898. vcpu_load(vcpu);
  1899. set_bit(irq->irq, vcpu->irq_pending);
  1900. set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
  1901. vcpu_put(vcpu);
  1902. return 0;
  1903. }
  1904. static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
  1905. struct kvm_debug_guest *dbg)
  1906. {
  1907. int r;
  1908. vcpu_load(vcpu);
  1909. r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
  1910. vcpu_put(vcpu);
  1911. return r;
  1912. }
  1913. static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
  1914. unsigned long address,
  1915. int *type)
  1916. {
  1917. struct kvm_vcpu *vcpu = vma->vm_file->private_data;
  1918. unsigned long pgoff;
  1919. struct page *page;
  1920. *type = VM_FAULT_MINOR;
  1921. pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  1922. if (pgoff == 0)
  1923. page = virt_to_page(vcpu->run);
  1924. else if (pgoff == KVM_PIO_PAGE_OFFSET)
  1925. page = virt_to_page(vcpu->pio_data);
  1926. else
  1927. return NOPAGE_SIGBUS;
  1928. get_page(page);
  1929. return page;
  1930. }
  1931. static struct vm_operations_struct kvm_vcpu_vm_ops = {
  1932. .nopage = kvm_vcpu_nopage,
  1933. };
  1934. static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
  1935. {
  1936. vma->vm_ops = &kvm_vcpu_vm_ops;
  1937. return 0;
  1938. }
  1939. static int kvm_vcpu_release(struct inode *inode, struct file *filp)
  1940. {
  1941. struct kvm_vcpu *vcpu = filp->private_data;
  1942. fput(vcpu->kvm->filp);
  1943. return 0;
  1944. }
  1945. static struct file_operations kvm_vcpu_fops = {
  1946. .release = kvm_vcpu_release,
  1947. .unlocked_ioctl = kvm_vcpu_ioctl,
  1948. .compat_ioctl = kvm_vcpu_ioctl,
  1949. .mmap = kvm_vcpu_mmap,
  1950. };
  1951. /*
  1952. * Allocates an inode for the vcpu.
  1953. */
  1954. static int create_vcpu_fd(struct kvm_vcpu *vcpu)
  1955. {
  1956. int fd, r;
  1957. struct inode *inode;
  1958. struct file *file;
  1959. r = anon_inode_getfd(&fd, &inode, &file,
  1960. "kvm-vcpu", &kvm_vcpu_fops, vcpu);
  1961. if (r)
  1962. return r;
  1963. atomic_inc(&vcpu->kvm->filp->f_count);
  1964. return fd;
  1965. }
  1966. /*
  1967. * Creates some virtual cpus. Good luck creating more than one.
  1968. */
  1969. static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
  1970. {
  1971. int r;
  1972. struct kvm_vcpu *vcpu;
  1973. struct page *page;
  1974. r = -EINVAL;
  1975. if (!valid_vcpu(n))
  1976. goto out;
  1977. vcpu = &kvm->vcpus[n];
  1978. mutex_lock(&vcpu->mutex);
  1979. if (vcpu->vmcs) {
  1980. mutex_unlock(&vcpu->mutex);
  1981. return -EEXIST;
  1982. }
  1983. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  1984. r = -ENOMEM;
  1985. if (!page)
  1986. goto out_unlock;
  1987. vcpu->run = page_address(page);
  1988. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  1989. r = -ENOMEM;
  1990. if (!page)
  1991. goto out_free_run;
  1992. vcpu->pio_data = page_address(page);
  1993. vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
  1994. FX_IMAGE_ALIGN);
  1995. vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
  1996. vcpu->cr0 = 0x10;
  1997. r = kvm_arch_ops->vcpu_create(vcpu);
  1998. if (r < 0)
  1999. goto out_free_vcpus;
  2000. r = kvm_mmu_create(vcpu);
  2001. if (r < 0)
  2002. goto out_free_vcpus;
  2003. kvm_arch_ops->vcpu_load(vcpu);
  2004. r = kvm_mmu_setup(vcpu);
  2005. if (r >= 0)
  2006. r = kvm_arch_ops->vcpu_setup(vcpu);
  2007. vcpu_put(vcpu);
  2008. if (r < 0)
  2009. goto out_free_vcpus;
  2010. r = create_vcpu_fd(vcpu);
  2011. if (r < 0)
  2012. goto out_free_vcpus;
  2013. spin_lock(&kvm_lock);
  2014. if (n >= kvm->nvcpus)
  2015. kvm->nvcpus = n + 1;
  2016. spin_unlock(&kvm_lock);
  2017. return r;
  2018. out_free_vcpus:
  2019. kvm_free_vcpu(vcpu);
  2020. out_free_run:
  2021. free_page((unsigned long)vcpu->run);
  2022. vcpu->run = NULL;
  2023. out_unlock:
  2024. mutex_unlock(&vcpu->mutex);
  2025. out:
  2026. return r;
  2027. }
  2028. static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
  2029. {
  2030. u64 efer;
  2031. int i;
  2032. struct kvm_cpuid_entry *e, *entry;
  2033. rdmsrl(MSR_EFER, efer);
  2034. entry = NULL;
  2035. for (i = 0; i < vcpu->cpuid_nent; ++i) {
  2036. e = &vcpu->cpuid_entries[i];
  2037. if (e->function == 0x80000001) {
  2038. entry = e;
  2039. break;
  2040. }
  2041. }
  2042. if (entry && (entry->edx & EFER_NX) && !(efer & EFER_NX)) {
  2043. entry->edx &= ~(1 << 20);
  2044. printk(KERN_INFO ": guest NX capability removed\n");
  2045. }
  2046. }
  2047. static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
  2048. struct kvm_cpuid *cpuid,
  2049. struct kvm_cpuid_entry __user *entries)
  2050. {
  2051. int r;
  2052. r = -E2BIG;
  2053. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  2054. goto out;
  2055. r = -EFAULT;
  2056. if (copy_from_user(&vcpu->cpuid_entries, entries,
  2057. cpuid->nent * sizeof(struct kvm_cpuid_entry)))
  2058. goto out;
  2059. vcpu->cpuid_nent = cpuid->nent;
  2060. cpuid_fix_nx_cap(vcpu);
  2061. return 0;
  2062. out:
  2063. return r;
  2064. }
  2065. static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
  2066. {
  2067. if (sigset) {
  2068. sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
  2069. vcpu->sigset_active = 1;
  2070. vcpu->sigset = *sigset;
  2071. } else
  2072. vcpu->sigset_active = 0;
  2073. return 0;
  2074. }
  2075. /*
  2076. * fxsave fpu state. Taken from x86_64/processor.h. To be killed when
  2077. * we have asm/x86/processor.h
  2078. */
  2079. struct fxsave {
  2080. u16 cwd;
  2081. u16 swd;
  2082. u16 twd;
  2083. u16 fop;
  2084. u64 rip;
  2085. u64 rdp;
  2086. u32 mxcsr;
  2087. u32 mxcsr_mask;
  2088. u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */
  2089. #ifdef CONFIG_X86_64
  2090. u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */
  2091. #else
  2092. u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */
  2093. #endif
  2094. };
  2095. static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  2096. {
  2097. struct fxsave *fxsave = (struct fxsave *)vcpu->guest_fx_image;
  2098. vcpu_load(vcpu);
  2099. memcpy(fpu->fpr, fxsave->st_space, 128);
  2100. fpu->fcw = fxsave->cwd;
  2101. fpu->fsw = fxsave->swd;
  2102. fpu->ftwx = fxsave->twd;
  2103. fpu->last_opcode = fxsave->fop;
  2104. fpu->last_ip = fxsave->rip;
  2105. fpu->last_dp = fxsave->rdp;
  2106. memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
  2107. vcpu_put(vcpu);
  2108. return 0;
  2109. }
  2110. static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  2111. {
  2112. struct fxsave *fxsave = (struct fxsave *)vcpu->guest_fx_image;
  2113. vcpu_load(vcpu);
  2114. memcpy(fxsave->st_space, fpu->fpr, 128);
  2115. fxsave->cwd = fpu->fcw;
  2116. fxsave->swd = fpu->fsw;
  2117. fxsave->twd = fpu->ftwx;
  2118. fxsave->fop = fpu->last_opcode;
  2119. fxsave->rip = fpu->last_ip;
  2120. fxsave->rdp = fpu->last_dp;
  2121. memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
  2122. vcpu_put(vcpu);
  2123. return 0;
  2124. }
  2125. static long kvm_vcpu_ioctl(struct file *filp,
  2126. unsigned int ioctl, unsigned long arg)
  2127. {
  2128. struct kvm_vcpu *vcpu = filp->private_data;
  2129. void __user *argp = (void __user *)arg;
  2130. int r = -EINVAL;
  2131. switch (ioctl) {
  2132. case KVM_RUN:
  2133. r = -EINVAL;
  2134. if (arg)
  2135. goto out;
  2136. r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
  2137. break;
  2138. case KVM_GET_REGS: {
  2139. struct kvm_regs kvm_regs;
  2140. memset(&kvm_regs, 0, sizeof kvm_regs);
  2141. r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
  2142. if (r)
  2143. goto out;
  2144. r = -EFAULT;
  2145. if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
  2146. goto out;
  2147. r = 0;
  2148. break;
  2149. }
  2150. case KVM_SET_REGS: {
  2151. struct kvm_regs kvm_regs;
  2152. r = -EFAULT;
  2153. if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
  2154. goto out;
  2155. r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
  2156. if (r)
  2157. goto out;
  2158. r = 0;
  2159. break;
  2160. }
  2161. case KVM_GET_SREGS: {
  2162. struct kvm_sregs kvm_sregs;
  2163. memset(&kvm_sregs, 0, sizeof kvm_sregs);
  2164. r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
  2165. if (r)
  2166. goto out;
  2167. r = -EFAULT;
  2168. if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
  2169. goto out;
  2170. r = 0;
  2171. break;
  2172. }
  2173. case KVM_SET_SREGS: {
  2174. struct kvm_sregs kvm_sregs;
  2175. r = -EFAULT;
  2176. if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
  2177. goto out;
  2178. r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
  2179. if (r)
  2180. goto out;
  2181. r = 0;
  2182. break;
  2183. }
  2184. case KVM_TRANSLATE: {
  2185. struct kvm_translation tr;
  2186. r = -EFAULT;
  2187. if (copy_from_user(&tr, argp, sizeof tr))
  2188. goto out;
  2189. r = kvm_vcpu_ioctl_translate(vcpu, &tr);
  2190. if (r)
  2191. goto out;
  2192. r = -EFAULT;
  2193. if (copy_to_user(argp, &tr, sizeof tr))
  2194. goto out;
  2195. r = 0;
  2196. break;
  2197. }
  2198. case KVM_INTERRUPT: {
  2199. struct kvm_interrupt irq;
  2200. r = -EFAULT;
  2201. if (copy_from_user(&irq, argp, sizeof irq))
  2202. goto out;
  2203. r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
  2204. if (r)
  2205. goto out;
  2206. r = 0;
  2207. break;
  2208. }
  2209. case KVM_DEBUG_GUEST: {
  2210. struct kvm_debug_guest dbg;
  2211. r = -EFAULT;
  2212. if (copy_from_user(&dbg, argp, sizeof dbg))
  2213. goto out;
  2214. r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
  2215. if (r)
  2216. goto out;
  2217. r = 0;
  2218. break;
  2219. }
  2220. case KVM_GET_MSRS:
  2221. r = msr_io(vcpu, argp, kvm_get_msr, 1);
  2222. break;
  2223. case KVM_SET_MSRS:
  2224. r = msr_io(vcpu, argp, do_set_msr, 0);
  2225. break;
  2226. case KVM_SET_CPUID: {
  2227. struct kvm_cpuid __user *cpuid_arg = argp;
  2228. struct kvm_cpuid cpuid;
  2229. r = -EFAULT;
  2230. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  2231. goto out;
  2232. r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
  2233. if (r)
  2234. goto out;
  2235. break;
  2236. }
  2237. case KVM_SET_SIGNAL_MASK: {
  2238. struct kvm_signal_mask __user *sigmask_arg = argp;
  2239. struct kvm_signal_mask kvm_sigmask;
  2240. sigset_t sigset, *p;
  2241. p = NULL;
  2242. if (argp) {
  2243. r = -EFAULT;
  2244. if (copy_from_user(&kvm_sigmask, argp,
  2245. sizeof kvm_sigmask))
  2246. goto out;
  2247. r = -EINVAL;
  2248. if (kvm_sigmask.len != sizeof sigset)
  2249. goto out;
  2250. r = -EFAULT;
  2251. if (copy_from_user(&sigset, sigmask_arg->sigset,
  2252. sizeof sigset))
  2253. goto out;
  2254. p = &sigset;
  2255. }
  2256. r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
  2257. break;
  2258. }
  2259. case KVM_GET_FPU: {
  2260. struct kvm_fpu fpu;
  2261. memset(&fpu, 0, sizeof fpu);
  2262. r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu);
  2263. if (r)
  2264. goto out;
  2265. r = -EFAULT;
  2266. if (copy_to_user(argp, &fpu, sizeof fpu))
  2267. goto out;
  2268. r = 0;
  2269. break;
  2270. }
  2271. case KVM_SET_FPU: {
  2272. struct kvm_fpu fpu;
  2273. r = -EFAULT;
  2274. if (copy_from_user(&fpu, argp, sizeof fpu))
  2275. goto out;
  2276. r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu);
  2277. if (r)
  2278. goto out;
  2279. r = 0;
  2280. break;
  2281. }
  2282. default:
  2283. ;
  2284. }
  2285. out:
  2286. return r;
  2287. }
  2288. static long kvm_vm_ioctl(struct file *filp,
  2289. unsigned int ioctl, unsigned long arg)
  2290. {
  2291. struct kvm *kvm = filp->private_data;
  2292. void __user *argp = (void __user *)arg;
  2293. int r = -EINVAL;
  2294. switch (ioctl) {
  2295. case KVM_CREATE_VCPU:
  2296. r = kvm_vm_ioctl_create_vcpu(kvm, arg);
  2297. if (r < 0)
  2298. goto out;
  2299. break;
  2300. case KVM_SET_MEMORY_REGION: {
  2301. struct kvm_memory_region kvm_mem;
  2302. r = -EFAULT;
  2303. if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
  2304. goto out;
  2305. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
  2306. if (r)
  2307. goto out;
  2308. break;
  2309. }
  2310. case KVM_GET_DIRTY_LOG: {
  2311. struct kvm_dirty_log log;
  2312. r = -EFAULT;
  2313. if (copy_from_user(&log, argp, sizeof log))
  2314. goto out;
  2315. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  2316. if (r)
  2317. goto out;
  2318. break;
  2319. }
  2320. case KVM_SET_MEMORY_ALIAS: {
  2321. struct kvm_memory_alias alias;
  2322. r = -EFAULT;
  2323. if (copy_from_user(&alias, argp, sizeof alias))
  2324. goto out;
  2325. r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
  2326. if (r)
  2327. goto out;
  2328. break;
  2329. }
  2330. default:
  2331. ;
  2332. }
  2333. out:
  2334. return r;
  2335. }
  2336. static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
  2337. unsigned long address,
  2338. int *type)
  2339. {
  2340. struct kvm *kvm = vma->vm_file->private_data;
  2341. unsigned long pgoff;
  2342. struct page *page;
  2343. *type = VM_FAULT_MINOR;
  2344. pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  2345. page = gfn_to_page(kvm, pgoff);
  2346. if (!page)
  2347. return NOPAGE_SIGBUS;
  2348. get_page(page);
  2349. return page;
  2350. }
  2351. static struct vm_operations_struct kvm_vm_vm_ops = {
  2352. .nopage = kvm_vm_nopage,
  2353. };
  2354. static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
  2355. {
  2356. vma->vm_ops = &kvm_vm_vm_ops;
  2357. return 0;
  2358. }
  2359. static struct file_operations kvm_vm_fops = {
  2360. .release = kvm_vm_release,
  2361. .unlocked_ioctl = kvm_vm_ioctl,
  2362. .compat_ioctl = kvm_vm_ioctl,
  2363. .mmap = kvm_vm_mmap,
  2364. };
  2365. static int kvm_dev_ioctl_create_vm(void)
  2366. {
  2367. int fd, r;
  2368. struct inode *inode;
  2369. struct file *file;
  2370. struct kvm *kvm;
  2371. kvm = kvm_create_vm();
  2372. if (IS_ERR(kvm))
  2373. return PTR_ERR(kvm);
  2374. r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
  2375. if (r) {
  2376. kvm_destroy_vm(kvm);
  2377. return r;
  2378. }
  2379. kvm->filp = file;
  2380. return fd;
  2381. }
  2382. static long kvm_dev_ioctl(struct file *filp,
  2383. unsigned int ioctl, unsigned long arg)
  2384. {
  2385. void __user *argp = (void __user *)arg;
  2386. long r = -EINVAL;
  2387. switch (ioctl) {
  2388. case KVM_GET_API_VERSION:
  2389. r = -EINVAL;
  2390. if (arg)
  2391. goto out;
  2392. r = KVM_API_VERSION;
  2393. break;
  2394. case KVM_CREATE_VM:
  2395. r = -EINVAL;
  2396. if (arg)
  2397. goto out;
  2398. r = kvm_dev_ioctl_create_vm();
  2399. break;
  2400. case KVM_GET_MSR_INDEX_LIST: {
  2401. struct kvm_msr_list __user *user_msr_list = argp;
  2402. struct kvm_msr_list msr_list;
  2403. unsigned n;
  2404. r = -EFAULT;
  2405. if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
  2406. goto out;
  2407. n = msr_list.nmsrs;
  2408. msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
  2409. if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
  2410. goto out;
  2411. r = -E2BIG;
  2412. if (n < num_msrs_to_save)
  2413. goto out;
  2414. r = -EFAULT;
  2415. if (copy_to_user(user_msr_list->indices, &msrs_to_save,
  2416. num_msrs_to_save * sizeof(u32)))
  2417. goto out;
  2418. if (copy_to_user(user_msr_list->indices
  2419. + num_msrs_to_save * sizeof(u32),
  2420. &emulated_msrs,
  2421. ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
  2422. goto out;
  2423. r = 0;
  2424. break;
  2425. }
  2426. case KVM_CHECK_EXTENSION:
  2427. /*
  2428. * No extensions defined at present.
  2429. */
  2430. r = 0;
  2431. break;
  2432. case KVM_GET_VCPU_MMAP_SIZE:
  2433. r = -EINVAL;
  2434. if (arg)
  2435. goto out;
  2436. r = 2 * PAGE_SIZE;
  2437. break;
  2438. default:
  2439. ;
  2440. }
  2441. out:
  2442. return r;
  2443. }
  2444. static struct file_operations kvm_chardev_ops = {
  2445. .open = kvm_dev_open,
  2446. .release = kvm_dev_release,
  2447. .unlocked_ioctl = kvm_dev_ioctl,
  2448. .compat_ioctl = kvm_dev_ioctl,
  2449. };
  2450. static struct miscdevice kvm_dev = {
  2451. KVM_MINOR,
  2452. "kvm",
  2453. &kvm_chardev_ops,
  2454. };
  2455. static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
  2456. void *v)
  2457. {
  2458. if (val == SYS_RESTART) {
  2459. /*
  2460. * Some (well, at least mine) BIOSes hang on reboot if
  2461. * in vmx root mode.
  2462. */
  2463. printk(KERN_INFO "kvm: exiting hardware virtualization\n");
  2464. on_each_cpu(hardware_disable, NULL, 0, 1);
  2465. }
  2466. return NOTIFY_OK;
  2467. }
  2468. static struct notifier_block kvm_reboot_notifier = {
  2469. .notifier_call = kvm_reboot,
  2470. .priority = 0,
  2471. };
  2472. /*
  2473. * Make sure that a cpu that is being hot-unplugged does not have any vcpus
  2474. * cached on it.
  2475. */
  2476. static void decache_vcpus_on_cpu(int cpu)
  2477. {
  2478. struct kvm *vm;
  2479. struct kvm_vcpu *vcpu;
  2480. int i;
  2481. spin_lock(&kvm_lock);
  2482. list_for_each_entry(vm, &vm_list, vm_list)
  2483. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  2484. vcpu = &vm->vcpus[i];
  2485. /*
  2486. * If the vcpu is locked, then it is running on some
  2487. * other cpu and therefore it is not cached on the
  2488. * cpu in question.
  2489. *
  2490. * If it's not locked, check the last cpu it executed
  2491. * on.
  2492. */
  2493. if (mutex_trylock(&vcpu->mutex)) {
  2494. if (vcpu->cpu == cpu) {
  2495. kvm_arch_ops->vcpu_decache(vcpu);
  2496. vcpu->cpu = -1;
  2497. }
  2498. mutex_unlock(&vcpu->mutex);
  2499. }
  2500. }
  2501. spin_unlock(&kvm_lock);
  2502. }
  2503. static void hardware_enable(void *junk)
  2504. {
  2505. int cpu = raw_smp_processor_id();
  2506. if (cpu_isset(cpu, cpus_hardware_enabled))
  2507. return;
  2508. cpu_set(cpu, cpus_hardware_enabled);
  2509. kvm_arch_ops->hardware_enable(NULL);
  2510. }
  2511. static void hardware_disable(void *junk)
  2512. {
  2513. int cpu = raw_smp_processor_id();
  2514. if (!cpu_isset(cpu, cpus_hardware_enabled))
  2515. return;
  2516. cpu_clear(cpu, cpus_hardware_enabled);
  2517. decache_vcpus_on_cpu(cpu);
  2518. kvm_arch_ops->hardware_disable(NULL);
  2519. }
  2520. static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  2521. void *v)
  2522. {
  2523. int cpu = (long)v;
  2524. switch (val) {
  2525. case CPU_DYING:
  2526. case CPU_DYING_FROZEN:
  2527. case CPU_UP_CANCELED:
  2528. case CPU_UP_CANCELED_FROZEN:
  2529. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  2530. cpu);
  2531. smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
  2532. break;
  2533. case CPU_ONLINE:
  2534. case CPU_ONLINE_FROZEN:
  2535. printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
  2536. cpu);
  2537. smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
  2538. break;
  2539. }
  2540. return NOTIFY_OK;
  2541. }
  2542. void kvm_io_bus_init(struct kvm_io_bus *bus)
  2543. {
  2544. memset(bus, 0, sizeof(*bus));
  2545. }
  2546. void kvm_io_bus_destroy(struct kvm_io_bus *bus)
  2547. {
  2548. int i;
  2549. for (i = 0; i < bus->dev_count; i++) {
  2550. struct kvm_io_device *pos = bus->devs[i];
  2551. kvm_iodevice_destructor(pos);
  2552. }
  2553. }
  2554. struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
  2555. {
  2556. int i;
  2557. for (i = 0; i < bus->dev_count; i++) {
  2558. struct kvm_io_device *pos = bus->devs[i];
  2559. if (pos->in_range(pos, addr))
  2560. return pos;
  2561. }
  2562. return NULL;
  2563. }
  2564. void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
  2565. {
  2566. BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
  2567. bus->devs[bus->dev_count++] = dev;
  2568. }
  2569. static struct notifier_block kvm_cpu_notifier = {
  2570. .notifier_call = kvm_cpu_hotplug,
  2571. .priority = 20, /* must be > scheduler priority */
  2572. };
  2573. static u64 stat_get(void *_offset)
  2574. {
  2575. unsigned offset = (long)_offset;
  2576. u64 total = 0;
  2577. struct kvm *kvm;
  2578. struct kvm_vcpu *vcpu;
  2579. int i;
  2580. spin_lock(&kvm_lock);
  2581. list_for_each_entry(kvm, &vm_list, vm_list)
  2582. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  2583. vcpu = &kvm->vcpus[i];
  2584. total += *(u32 *)((void *)vcpu + offset);
  2585. }
  2586. spin_unlock(&kvm_lock);
  2587. return total;
  2588. }
  2589. static void stat_set(void *offset, u64 val)
  2590. {
  2591. }
  2592. DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, stat_set, "%llu\n");
  2593. static __init void kvm_init_debug(void)
  2594. {
  2595. struct kvm_stats_debugfs_item *p;
  2596. debugfs_dir = debugfs_create_dir("kvm", NULL);
  2597. for (p = debugfs_entries; p->name; ++p)
  2598. p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
  2599. (void *)(long)p->offset,
  2600. &stat_fops);
  2601. }
  2602. static void kvm_exit_debug(void)
  2603. {
  2604. struct kvm_stats_debugfs_item *p;
  2605. for (p = debugfs_entries; p->name; ++p)
  2606. debugfs_remove(p->dentry);
  2607. debugfs_remove(debugfs_dir);
  2608. }
  2609. static int kvm_suspend(struct sys_device *dev, pm_message_t state)
  2610. {
  2611. hardware_disable(NULL);
  2612. return 0;
  2613. }
  2614. static int kvm_resume(struct sys_device *dev)
  2615. {
  2616. hardware_enable(NULL);
  2617. return 0;
  2618. }
  2619. static struct sysdev_class kvm_sysdev_class = {
  2620. set_kset_name("kvm"),
  2621. .suspend = kvm_suspend,
  2622. .resume = kvm_resume,
  2623. };
  2624. static struct sys_device kvm_sysdev = {
  2625. .id = 0,
  2626. .cls = &kvm_sysdev_class,
  2627. };
  2628. hpa_t bad_page_address;
  2629. int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
  2630. {
  2631. int r;
  2632. if (kvm_arch_ops) {
  2633. printk(KERN_ERR "kvm: already loaded the other module\n");
  2634. return -EEXIST;
  2635. }
  2636. if (!ops->cpu_has_kvm_support()) {
  2637. printk(KERN_ERR "kvm: no hardware support\n");
  2638. return -EOPNOTSUPP;
  2639. }
  2640. if (ops->disabled_by_bios()) {
  2641. printk(KERN_ERR "kvm: disabled by bios\n");
  2642. return -EOPNOTSUPP;
  2643. }
  2644. kvm_arch_ops = ops;
  2645. r = kvm_arch_ops->hardware_setup();
  2646. if (r < 0)
  2647. goto out;
  2648. on_each_cpu(hardware_enable, NULL, 0, 1);
  2649. r = register_cpu_notifier(&kvm_cpu_notifier);
  2650. if (r)
  2651. goto out_free_1;
  2652. register_reboot_notifier(&kvm_reboot_notifier);
  2653. r = sysdev_class_register(&kvm_sysdev_class);
  2654. if (r)
  2655. goto out_free_2;
  2656. r = sysdev_register(&kvm_sysdev);
  2657. if (r)
  2658. goto out_free_3;
  2659. kvm_chardev_ops.owner = module;
  2660. r = misc_register(&kvm_dev);
  2661. if (r) {
  2662. printk (KERN_ERR "kvm: misc device register failed\n");
  2663. goto out_free;
  2664. }
  2665. return r;
  2666. out_free:
  2667. sysdev_unregister(&kvm_sysdev);
  2668. out_free_3:
  2669. sysdev_class_unregister(&kvm_sysdev_class);
  2670. out_free_2:
  2671. unregister_reboot_notifier(&kvm_reboot_notifier);
  2672. unregister_cpu_notifier(&kvm_cpu_notifier);
  2673. out_free_1:
  2674. on_each_cpu(hardware_disable, NULL, 0, 1);
  2675. kvm_arch_ops->hardware_unsetup();
  2676. out:
  2677. kvm_arch_ops = NULL;
  2678. return r;
  2679. }
  2680. void kvm_exit_arch(void)
  2681. {
  2682. misc_deregister(&kvm_dev);
  2683. sysdev_unregister(&kvm_sysdev);
  2684. sysdev_class_unregister(&kvm_sysdev_class);
  2685. unregister_reboot_notifier(&kvm_reboot_notifier);
  2686. unregister_cpu_notifier(&kvm_cpu_notifier);
  2687. on_each_cpu(hardware_disable, NULL, 0, 1);
  2688. kvm_arch_ops->hardware_unsetup();
  2689. kvm_arch_ops = NULL;
  2690. }
  2691. static __init int kvm_init(void)
  2692. {
  2693. static struct page *bad_page;
  2694. int r;
  2695. r = kvm_mmu_module_init();
  2696. if (r)
  2697. goto out4;
  2698. kvm_init_debug();
  2699. kvm_init_msr_list();
  2700. if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
  2701. r = -ENOMEM;
  2702. goto out;
  2703. }
  2704. bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
  2705. memset(__va(bad_page_address), 0, PAGE_SIZE);
  2706. return 0;
  2707. out:
  2708. kvm_exit_debug();
  2709. kvm_mmu_module_exit();
  2710. out4:
  2711. return r;
  2712. }
  2713. static __exit void kvm_exit(void)
  2714. {
  2715. kvm_exit_debug();
  2716. __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
  2717. kvm_mmu_module_exit();
  2718. }
  2719. module_init(kvm_init)
  2720. module_exit(kvm_exit)
  2721. EXPORT_SYMBOL_GPL(kvm_init_arch);
  2722. EXPORT_SYMBOL_GPL(kvm_exit_arch);