x86.c 128 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * derived from drivers/kvm/kvm_main.c
  5. *
  6. * Copyright (C) 2006 Qumranet, Inc.
  7. * Copyright (C) 2008 Qumranet, Inc.
  8. * Copyright IBM Corporation, 2008
  9. *
  10. * Authors:
  11. * Avi Kivity <avi@qumranet.com>
  12. * Yaniv Kamay <yaniv@qumranet.com>
  13. * Amit Shah <amit.shah@qumranet.com>
  14. * Ben-Ami Yassour <benami@il.ibm.com>
  15. *
  16. * This work is licensed under the terms of the GNU GPL, version 2. See
  17. * the COPYING file in the top-level directory.
  18. *
  19. */
  20. #include <linux/kvm_host.h>
  21. #include "irq.h"
  22. #include "mmu.h"
  23. #include "i8254.h"
  24. #include "tss.h"
  25. #include "kvm_cache_regs.h"
  26. #include "x86.h"
  27. #include <linux/clocksource.h>
  28. #include <linux/interrupt.h>
  29. #include <linux/kvm.h>
  30. #include <linux/fs.h>
  31. #include <linux/vmalloc.h>
  32. #include <linux/module.h>
  33. #include <linux/mman.h>
  34. #include <linux/highmem.h>
  35. #include <linux/iommu.h>
  36. #include <linux/intel-iommu.h>
  37. #include <linux/cpufreq.h>
  38. #include <linux/user-return-notifier.h>
  39. #include <trace/events/kvm.h>
  40. #undef TRACE_INCLUDE_FILE
  41. #define CREATE_TRACE_POINTS
  42. #include "trace.h"
  43. #include <asm/debugreg.h>
  44. #include <asm/uaccess.h>
  45. #include <asm/msr.h>
  46. #include <asm/desc.h>
  47. #include <asm/mtrr.h>
  48. #include <asm/mce.h>
  49. #define MAX_IO_MSRS 256
  50. #define CR0_RESERVED_BITS \
  51. (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
  52. | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
  53. | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
  54. #define CR4_RESERVED_BITS \
  55. (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
  56. | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
  57. | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \
  58. | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
  59. #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
  60. #define KVM_MAX_MCE_BANKS 32
  61. #define KVM_MCE_CAP_SUPPORTED MCG_CTL_P
  62. /* EFER defaults:
  63. * - enable syscall per default because its emulated by KVM
  64. * - enable LME and LMA per default on 64 bit KVM
  65. */
  66. #ifdef CONFIG_X86_64
  67. static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffafeULL;
  68. #else
  69. static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffffeULL;
  70. #endif
  71. #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  72. #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  73. static void update_cr8_intercept(struct kvm_vcpu *vcpu);
  74. static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  75. struct kvm_cpuid_entry2 __user *entries);
  76. struct kvm_x86_ops *kvm_x86_ops;
  77. EXPORT_SYMBOL_GPL(kvm_x86_ops);
  78. int ignore_msrs = 0;
  79. module_param_named(ignore_msrs, ignore_msrs, bool, S_IRUGO | S_IWUSR);
  80. #define KVM_NR_SHARED_MSRS 16
  81. struct kvm_shared_msrs_global {
  82. int nr;
  83. struct kvm_shared_msr {
  84. u32 msr;
  85. u64 value;
  86. } msrs[KVM_NR_SHARED_MSRS];
  87. };
  88. struct kvm_shared_msrs {
  89. struct user_return_notifier urn;
  90. bool registered;
  91. u64 current_value[KVM_NR_SHARED_MSRS];
  92. };
  93. static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
  94. static DEFINE_PER_CPU(struct kvm_shared_msrs, shared_msrs);
  95. struct kvm_stats_debugfs_item debugfs_entries[] = {
  96. { "pf_fixed", VCPU_STAT(pf_fixed) },
  97. { "pf_guest", VCPU_STAT(pf_guest) },
  98. { "tlb_flush", VCPU_STAT(tlb_flush) },
  99. { "invlpg", VCPU_STAT(invlpg) },
  100. { "exits", VCPU_STAT(exits) },
  101. { "io_exits", VCPU_STAT(io_exits) },
  102. { "mmio_exits", VCPU_STAT(mmio_exits) },
  103. { "signal_exits", VCPU_STAT(signal_exits) },
  104. { "irq_window", VCPU_STAT(irq_window_exits) },
  105. { "nmi_window", VCPU_STAT(nmi_window_exits) },
  106. { "halt_exits", VCPU_STAT(halt_exits) },
  107. { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  108. { "hypercalls", VCPU_STAT(hypercalls) },
  109. { "request_irq", VCPU_STAT(request_irq_exits) },
  110. { "irq_exits", VCPU_STAT(irq_exits) },
  111. { "host_state_reload", VCPU_STAT(host_state_reload) },
  112. { "efer_reload", VCPU_STAT(efer_reload) },
  113. { "fpu_reload", VCPU_STAT(fpu_reload) },
  114. { "insn_emulation", VCPU_STAT(insn_emulation) },
  115. { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
  116. { "irq_injections", VCPU_STAT(irq_injections) },
  117. { "nmi_injections", VCPU_STAT(nmi_injections) },
  118. { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
  119. { "mmu_pte_write", VM_STAT(mmu_pte_write) },
  120. { "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
  121. { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
  122. { "mmu_flooded", VM_STAT(mmu_flooded) },
  123. { "mmu_recycled", VM_STAT(mmu_recycled) },
  124. { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
  125. { "mmu_unsync", VM_STAT(mmu_unsync) },
  126. { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
  127. { "largepages", VM_STAT(lpages) },
  128. { NULL }
  129. };
  130. static void kvm_on_user_return(struct user_return_notifier *urn)
  131. {
  132. unsigned slot;
  133. struct kvm_shared_msr *global;
  134. struct kvm_shared_msrs *locals
  135. = container_of(urn, struct kvm_shared_msrs, urn);
  136. for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
  137. global = &shared_msrs_global.msrs[slot];
  138. if (global->value != locals->current_value[slot]) {
  139. wrmsrl(global->msr, global->value);
  140. locals->current_value[slot] = global->value;
  141. }
  142. }
  143. locals->registered = false;
  144. user_return_notifier_unregister(urn);
  145. }
  146. void kvm_define_shared_msr(unsigned slot, u32 msr)
  147. {
  148. int cpu;
  149. u64 value;
  150. if (slot >= shared_msrs_global.nr)
  151. shared_msrs_global.nr = slot + 1;
  152. shared_msrs_global.msrs[slot].msr = msr;
  153. rdmsrl_safe(msr, &value);
  154. shared_msrs_global.msrs[slot].value = value;
  155. for_each_online_cpu(cpu)
  156. per_cpu(shared_msrs, cpu).current_value[slot] = value;
  157. }
  158. EXPORT_SYMBOL_GPL(kvm_define_shared_msr);
  159. static void kvm_shared_msr_cpu_online(void)
  160. {
  161. unsigned i;
  162. struct kvm_shared_msrs *locals = &__get_cpu_var(shared_msrs);
  163. for (i = 0; i < shared_msrs_global.nr; ++i)
  164. locals->current_value[i] = shared_msrs_global.msrs[i].value;
  165. }
  166. void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
  167. {
  168. struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);
  169. if (((value ^ smsr->current_value[slot]) & mask) == 0)
  170. return;
  171. smsr->current_value[slot] = value;
  172. wrmsrl(shared_msrs_global.msrs[slot].msr, value);
  173. if (!smsr->registered) {
  174. smsr->urn.on_user_return = kvm_on_user_return;
  175. user_return_notifier_register(&smsr->urn);
  176. smsr->registered = true;
  177. }
  178. }
  179. EXPORT_SYMBOL_GPL(kvm_set_shared_msr);
  180. static void drop_user_return_notifiers(void *ignore)
  181. {
  182. struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);
  183. if (smsr->registered)
  184. kvm_on_user_return(&smsr->urn);
  185. }
  186. unsigned long segment_base(u16 selector)
  187. {
  188. struct descriptor_table gdt;
  189. struct desc_struct *d;
  190. unsigned long table_base;
  191. unsigned long v;
  192. if (selector == 0)
  193. return 0;
  194. kvm_get_gdt(&gdt);
  195. table_base = gdt.base;
  196. if (selector & 4) { /* from ldt */
  197. u16 ldt_selector = kvm_read_ldt();
  198. table_base = segment_base(ldt_selector);
  199. }
  200. d = (struct desc_struct *)(table_base + (selector & ~7));
  201. v = get_desc_base(d);
  202. #ifdef CONFIG_X86_64
  203. if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
  204. v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
  205. #endif
  206. return v;
  207. }
  208. EXPORT_SYMBOL_GPL(segment_base);
  209. u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
  210. {
  211. if (irqchip_in_kernel(vcpu->kvm))
  212. return vcpu->arch.apic_base;
  213. else
  214. return vcpu->arch.apic_base;
  215. }
  216. EXPORT_SYMBOL_GPL(kvm_get_apic_base);
  217. void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
  218. {
  219. /* TODO: reserve bits check */
  220. if (irqchip_in_kernel(vcpu->kvm))
  221. kvm_lapic_set_base(vcpu, data);
  222. else
  223. vcpu->arch.apic_base = data;
  224. }
  225. EXPORT_SYMBOL_GPL(kvm_set_apic_base);
  226. void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
  227. {
  228. WARN_ON(vcpu->arch.exception.pending);
  229. vcpu->arch.exception.pending = true;
  230. vcpu->arch.exception.has_error_code = false;
  231. vcpu->arch.exception.nr = nr;
  232. }
  233. EXPORT_SYMBOL_GPL(kvm_queue_exception);
  234. void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr,
  235. u32 error_code)
  236. {
  237. ++vcpu->stat.pf_guest;
  238. if (vcpu->arch.exception.pending) {
  239. switch(vcpu->arch.exception.nr) {
  240. case DF_VECTOR:
  241. /* triple fault -> shutdown */
  242. set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
  243. return;
  244. case PF_VECTOR:
  245. vcpu->arch.exception.nr = DF_VECTOR;
  246. vcpu->arch.exception.error_code = 0;
  247. return;
  248. default:
  249. /* replace previous exception with a new one in a hope
  250. that instruction re-execution will regenerate lost
  251. exception */
  252. vcpu->arch.exception.pending = false;
  253. break;
  254. }
  255. }
  256. vcpu->arch.cr2 = addr;
  257. kvm_queue_exception_e(vcpu, PF_VECTOR, error_code);
  258. }
  259. void kvm_inject_nmi(struct kvm_vcpu *vcpu)
  260. {
  261. vcpu->arch.nmi_pending = 1;
  262. }
  263. EXPORT_SYMBOL_GPL(kvm_inject_nmi);
  264. void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
  265. {
  266. WARN_ON(vcpu->arch.exception.pending);
  267. vcpu->arch.exception.pending = true;
  268. vcpu->arch.exception.has_error_code = true;
  269. vcpu->arch.exception.nr = nr;
  270. vcpu->arch.exception.error_code = error_code;
  271. }
  272. EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
  273. /*
  274. * Checks if cpl <= required_cpl; if true, return true. Otherwise queue
  275. * a #GP and return false.
  276. */
  277. bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
  278. {
  279. if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
  280. return true;
  281. kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
  282. return false;
  283. }
  284. EXPORT_SYMBOL_GPL(kvm_require_cpl);
  285. /*
  286. * Load the pae pdptrs. Return true is they are all valid.
  287. */
  288. int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
  289. {
  290. gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
  291. unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
  292. int i;
  293. int ret;
  294. u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
  295. ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte,
  296. offset * sizeof(u64), sizeof(pdpte));
  297. if (ret < 0) {
  298. ret = 0;
  299. goto out;
  300. }
  301. for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
  302. if (is_present_gpte(pdpte[i]) &&
  303. (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
  304. ret = 0;
  305. goto out;
  306. }
  307. }
  308. ret = 1;
  309. memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
  310. __set_bit(VCPU_EXREG_PDPTR,
  311. (unsigned long *)&vcpu->arch.regs_avail);
  312. __set_bit(VCPU_EXREG_PDPTR,
  313. (unsigned long *)&vcpu->arch.regs_dirty);
  314. out:
  315. return ret;
  316. }
  317. EXPORT_SYMBOL_GPL(load_pdptrs);
  318. static bool pdptrs_changed(struct kvm_vcpu *vcpu)
  319. {
  320. u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
  321. bool changed = true;
  322. int r;
  323. if (is_long_mode(vcpu) || !is_pae(vcpu))
  324. return false;
  325. if (!test_bit(VCPU_EXREG_PDPTR,
  326. (unsigned long *)&vcpu->arch.regs_avail))
  327. return true;
  328. r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte));
  329. if (r < 0)
  330. goto out;
  331. changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0;
  332. out:
  333. return changed;
  334. }
  335. void kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  336. {
  337. if (cr0 & CR0_RESERVED_BITS) {
  338. printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
  339. cr0, vcpu->arch.cr0);
  340. kvm_inject_gp(vcpu, 0);
  341. return;
  342. }
  343. if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
  344. printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
  345. kvm_inject_gp(vcpu, 0);
  346. return;
  347. }
  348. if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
  349. printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
  350. "and a clear PE flag\n");
  351. kvm_inject_gp(vcpu, 0);
  352. return;
  353. }
  354. if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
  355. #ifdef CONFIG_X86_64
  356. if ((vcpu->arch.shadow_efer & EFER_LME)) {
  357. int cs_db, cs_l;
  358. if (!is_pae(vcpu)) {
  359. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  360. "in long mode while PAE is disabled\n");
  361. kvm_inject_gp(vcpu, 0);
  362. return;
  363. }
  364. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  365. if (cs_l) {
  366. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  367. "in long mode while CS.L == 1\n");
  368. kvm_inject_gp(vcpu, 0);
  369. return;
  370. }
  371. } else
  372. #endif
  373. if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
  374. printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
  375. "reserved bits\n");
  376. kvm_inject_gp(vcpu, 0);
  377. return;
  378. }
  379. }
  380. kvm_x86_ops->set_cr0(vcpu, cr0);
  381. vcpu->arch.cr0 = cr0;
  382. kvm_mmu_reset_context(vcpu);
  383. return;
  384. }
  385. EXPORT_SYMBOL_GPL(kvm_set_cr0);
  386. void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
  387. {
  388. kvm_set_cr0(vcpu, (vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f));
  389. }
  390. EXPORT_SYMBOL_GPL(kvm_lmsw);
  391. void kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  392. {
  393. unsigned long old_cr4 = vcpu->arch.cr4;
  394. unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE;
  395. if (cr4 & CR4_RESERVED_BITS) {
  396. printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
  397. kvm_inject_gp(vcpu, 0);
  398. return;
  399. }
  400. if (is_long_mode(vcpu)) {
  401. if (!(cr4 & X86_CR4_PAE)) {
  402. printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
  403. "in long mode\n");
  404. kvm_inject_gp(vcpu, 0);
  405. return;
  406. }
  407. } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
  408. && ((cr4 ^ old_cr4) & pdptr_bits)
  409. && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
  410. printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
  411. kvm_inject_gp(vcpu, 0);
  412. return;
  413. }
  414. if (cr4 & X86_CR4_VMXE) {
  415. printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
  416. kvm_inject_gp(vcpu, 0);
  417. return;
  418. }
  419. kvm_x86_ops->set_cr4(vcpu, cr4);
  420. vcpu->arch.cr4 = cr4;
  421. vcpu->arch.mmu.base_role.cr4_pge = (cr4 & X86_CR4_PGE) && !tdp_enabled;
  422. kvm_mmu_reset_context(vcpu);
  423. }
  424. EXPORT_SYMBOL_GPL(kvm_set_cr4);
  425. void kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
  426. {
  427. if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) {
  428. kvm_mmu_sync_roots(vcpu);
  429. kvm_mmu_flush_tlb(vcpu);
  430. return;
  431. }
  432. if (is_long_mode(vcpu)) {
  433. if (cr3 & CR3_L_MODE_RESERVED_BITS) {
  434. printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
  435. kvm_inject_gp(vcpu, 0);
  436. return;
  437. }
  438. } else {
  439. if (is_pae(vcpu)) {
  440. if (cr3 & CR3_PAE_RESERVED_BITS) {
  441. printk(KERN_DEBUG
  442. "set_cr3: #GP, reserved bits\n");
  443. kvm_inject_gp(vcpu, 0);
  444. return;
  445. }
  446. if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
  447. printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
  448. "reserved bits\n");
  449. kvm_inject_gp(vcpu, 0);
  450. return;
  451. }
  452. }
  453. /*
  454. * We don't check reserved bits in nonpae mode, because
  455. * this isn't enforced, and VMware depends on this.
  456. */
  457. }
  458. /*
  459. * Does the new cr3 value map to physical memory? (Note, we
  460. * catch an invalid cr3 even in real-mode, because it would
  461. * cause trouble later on when we turn on paging anyway.)
  462. *
  463. * A real CPU would silently accept an invalid cr3 and would
  464. * attempt to use it - with largely undefined (and often hard
  465. * to debug) behavior on the guest side.
  466. */
  467. if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
  468. kvm_inject_gp(vcpu, 0);
  469. else {
  470. vcpu->arch.cr3 = cr3;
  471. vcpu->arch.mmu.new_cr3(vcpu);
  472. }
  473. }
  474. EXPORT_SYMBOL_GPL(kvm_set_cr3);
  475. void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
  476. {
  477. if (cr8 & CR8_RESERVED_BITS) {
  478. printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
  479. kvm_inject_gp(vcpu, 0);
  480. return;
  481. }
  482. if (irqchip_in_kernel(vcpu->kvm))
  483. kvm_lapic_set_tpr(vcpu, cr8);
  484. else
  485. vcpu->arch.cr8 = cr8;
  486. }
  487. EXPORT_SYMBOL_GPL(kvm_set_cr8);
  488. unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
  489. {
  490. if (irqchip_in_kernel(vcpu->kvm))
  491. return kvm_lapic_get_cr8(vcpu);
  492. else
  493. return vcpu->arch.cr8;
  494. }
  495. EXPORT_SYMBOL_GPL(kvm_get_cr8);
  496. static inline u32 bit(int bitno)
  497. {
  498. return 1 << (bitno & 31);
  499. }
  500. /*
  501. * List of msr numbers which we expose to userspace through KVM_GET_MSRS
  502. * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
  503. *
  504. * This list is modified at module load time to reflect the
  505. * capabilities of the host cpu. This capabilities test skips MSRs that are
  506. * kvm-specific. Those are put in the beginning of the list.
  507. */
  508. #define KVM_SAVE_MSRS_BEGIN 2
  509. static u32 msrs_to_save[] = {
  510. MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
  511. MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  512. MSR_K6_STAR,
  513. #ifdef CONFIG_X86_64
  514. MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
  515. #endif
  516. MSR_IA32_TSC, MSR_IA32_PERF_STATUS, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA
  517. };
  518. static unsigned num_msrs_to_save;
  519. static u32 emulated_msrs[] = {
  520. MSR_IA32_MISC_ENABLE,
  521. };
  522. static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
  523. {
  524. if (efer & efer_reserved_bits) {
  525. printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
  526. efer);
  527. kvm_inject_gp(vcpu, 0);
  528. return;
  529. }
  530. if (is_paging(vcpu)
  531. && (vcpu->arch.shadow_efer & EFER_LME) != (efer & EFER_LME)) {
  532. printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
  533. kvm_inject_gp(vcpu, 0);
  534. return;
  535. }
  536. if (efer & EFER_FFXSR) {
  537. struct kvm_cpuid_entry2 *feat;
  538. feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
  539. if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT))) {
  540. printk(KERN_DEBUG "set_efer: #GP, enable FFXSR w/o CPUID capability\n");
  541. kvm_inject_gp(vcpu, 0);
  542. return;
  543. }
  544. }
  545. if (efer & EFER_SVME) {
  546. struct kvm_cpuid_entry2 *feat;
  547. feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
  548. if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM))) {
  549. printk(KERN_DEBUG "set_efer: #GP, enable SVM w/o SVM\n");
  550. kvm_inject_gp(vcpu, 0);
  551. return;
  552. }
  553. }
  554. kvm_x86_ops->set_efer(vcpu, efer);
  555. efer &= ~EFER_LMA;
  556. efer |= vcpu->arch.shadow_efer & EFER_LMA;
  557. vcpu->arch.shadow_efer = efer;
  558. vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
  559. kvm_mmu_reset_context(vcpu);
  560. }
  561. void kvm_enable_efer_bits(u64 mask)
  562. {
  563. efer_reserved_bits &= ~mask;
  564. }
  565. EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
  566. /*
  567. * Writes msr value into into the appropriate "register".
  568. * Returns 0 on success, non-0 otherwise.
  569. * Assumes vcpu_load() was already called.
  570. */
  571. int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
  572. {
  573. return kvm_x86_ops->set_msr(vcpu, msr_index, data);
  574. }
  575. /*
  576. * Adapt set_msr() to msr_io()'s calling convention
  577. */
  578. static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
  579. {
  580. return kvm_set_msr(vcpu, index, *data);
  581. }
  582. static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
  583. {
  584. static int version;
  585. struct pvclock_wall_clock wc;
  586. struct timespec boot;
  587. if (!wall_clock)
  588. return;
  589. version++;
  590. kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
  591. /*
  592. * The guest calculates current wall clock time by adding
  593. * system time (updated by kvm_write_guest_time below) to the
  594. * wall clock specified here. guest system time equals host
  595. * system time for us, thus we must fill in host boot time here.
  596. */
  597. getboottime(&boot);
  598. wc.sec = boot.tv_sec;
  599. wc.nsec = boot.tv_nsec;
  600. wc.version = version;
  601. kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
  602. version++;
  603. kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
  604. }
  605. static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
  606. {
  607. uint32_t quotient, remainder;
  608. /* Don't try to replace with do_div(), this one calculates
  609. * "(dividend << 32) / divisor" */
  610. __asm__ ( "divl %4"
  611. : "=a" (quotient), "=d" (remainder)
  612. : "0" (0), "1" (dividend), "r" (divisor) );
  613. return quotient;
  614. }
  615. static void kvm_set_time_scale(uint32_t tsc_khz, struct pvclock_vcpu_time_info *hv_clock)
  616. {
  617. uint64_t nsecs = 1000000000LL;
  618. int32_t shift = 0;
  619. uint64_t tps64;
  620. uint32_t tps32;
  621. tps64 = tsc_khz * 1000LL;
  622. while (tps64 > nsecs*2) {
  623. tps64 >>= 1;
  624. shift--;
  625. }
  626. tps32 = (uint32_t)tps64;
  627. while (tps32 <= (uint32_t)nsecs) {
  628. tps32 <<= 1;
  629. shift++;
  630. }
  631. hv_clock->tsc_shift = shift;
  632. hv_clock->tsc_to_system_mul = div_frac(nsecs, tps32);
  633. pr_debug("%s: tsc_khz %u, tsc_shift %d, tsc_mul %u\n",
  634. __func__, tsc_khz, hv_clock->tsc_shift,
  635. hv_clock->tsc_to_system_mul);
  636. }
  637. static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
  638. static void kvm_write_guest_time(struct kvm_vcpu *v)
  639. {
  640. struct timespec ts;
  641. unsigned long flags;
  642. struct kvm_vcpu_arch *vcpu = &v->arch;
  643. void *shared_kaddr;
  644. unsigned long this_tsc_khz;
  645. if ((!vcpu->time_page))
  646. return;
  647. this_tsc_khz = get_cpu_var(cpu_tsc_khz);
  648. if (unlikely(vcpu->hv_clock_tsc_khz != this_tsc_khz)) {
  649. kvm_set_time_scale(this_tsc_khz, &vcpu->hv_clock);
  650. vcpu->hv_clock_tsc_khz = this_tsc_khz;
  651. }
  652. put_cpu_var(cpu_tsc_khz);
  653. /* Keep irq disabled to prevent changes to the clock */
  654. local_irq_save(flags);
  655. kvm_get_msr(v, MSR_IA32_TSC, &vcpu->hv_clock.tsc_timestamp);
  656. ktime_get_ts(&ts);
  657. monotonic_to_bootbased(&ts);
  658. local_irq_restore(flags);
  659. /* With all the info we got, fill in the values */
  660. vcpu->hv_clock.system_time = ts.tv_nsec +
  661. (NSEC_PER_SEC * (u64)ts.tv_sec) + v->kvm->arch.kvmclock_offset;
  662. /*
  663. * The interface expects us to write an even number signaling that the
  664. * update is finished. Since the guest won't see the intermediate
  665. * state, we just increase by 2 at the end.
  666. */
  667. vcpu->hv_clock.version += 2;
  668. shared_kaddr = kmap_atomic(vcpu->time_page, KM_USER0);
  669. memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock,
  670. sizeof(vcpu->hv_clock));
  671. kunmap_atomic(shared_kaddr, KM_USER0);
  672. mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT);
  673. }
  674. static int kvm_request_guest_time_update(struct kvm_vcpu *v)
  675. {
  676. struct kvm_vcpu_arch *vcpu = &v->arch;
  677. if (!vcpu->time_page)
  678. return 0;
  679. set_bit(KVM_REQ_KVMCLOCK_UPDATE, &v->requests);
  680. return 1;
  681. }
  682. static bool msr_mtrr_valid(unsigned msr)
  683. {
  684. switch (msr) {
  685. case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
  686. case MSR_MTRRfix64K_00000:
  687. case MSR_MTRRfix16K_80000:
  688. case MSR_MTRRfix16K_A0000:
  689. case MSR_MTRRfix4K_C0000:
  690. case MSR_MTRRfix4K_C8000:
  691. case MSR_MTRRfix4K_D0000:
  692. case MSR_MTRRfix4K_D8000:
  693. case MSR_MTRRfix4K_E0000:
  694. case MSR_MTRRfix4K_E8000:
  695. case MSR_MTRRfix4K_F0000:
  696. case MSR_MTRRfix4K_F8000:
  697. case MSR_MTRRdefType:
  698. case MSR_IA32_CR_PAT:
  699. return true;
  700. case 0x2f8:
  701. return true;
  702. }
  703. return false;
  704. }
  705. static bool valid_pat_type(unsigned t)
  706. {
  707. return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
  708. }
  709. static bool valid_mtrr_type(unsigned t)
  710. {
  711. return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
  712. }
  713. static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  714. {
  715. int i;
  716. if (!msr_mtrr_valid(msr))
  717. return false;
  718. if (msr == MSR_IA32_CR_PAT) {
  719. for (i = 0; i < 8; i++)
  720. if (!valid_pat_type((data >> (i * 8)) & 0xff))
  721. return false;
  722. return true;
  723. } else if (msr == MSR_MTRRdefType) {
  724. if (data & ~0xcff)
  725. return false;
  726. return valid_mtrr_type(data & 0xff);
  727. } else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
  728. for (i = 0; i < 8 ; i++)
  729. if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
  730. return false;
  731. return true;
  732. }
  733. /* variable MTRRs */
  734. return valid_mtrr_type(data & 0xff);
  735. }
  736. static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  737. {
  738. u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
  739. if (!mtrr_valid(vcpu, msr, data))
  740. return 1;
  741. if (msr == MSR_MTRRdefType) {
  742. vcpu->arch.mtrr_state.def_type = data;
  743. vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
  744. } else if (msr == MSR_MTRRfix64K_00000)
  745. p[0] = data;
  746. else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
  747. p[1 + msr - MSR_MTRRfix16K_80000] = data;
  748. else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
  749. p[3 + msr - MSR_MTRRfix4K_C0000] = data;
  750. else if (msr == MSR_IA32_CR_PAT)
  751. vcpu->arch.pat = data;
  752. else { /* Variable MTRRs */
  753. int idx, is_mtrr_mask;
  754. u64 *pt;
  755. idx = (msr - 0x200) / 2;
  756. is_mtrr_mask = msr - 0x200 - 2 * idx;
  757. if (!is_mtrr_mask)
  758. pt =
  759. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
  760. else
  761. pt =
  762. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
  763. *pt = data;
  764. }
  765. kvm_mmu_reset_context(vcpu);
  766. return 0;
  767. }
  768. static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  769. {
  770. u64 mcg_cap = vcpu->arch.mcg_cap;
  771. unsigned bank_num = mcg_cap & 0xff;
  772. switch (msr) {
  773. case MSR_IA32_MCG_STATUS:
  774. vcpu->arch.mcg_status = data;
  775. break;
  776. case MSR_IA32_MCG_CTL:
  777. if (!(mcg_cap & MCG_CTL_P))
  778. return 1;
  779. if (data != 0 && data != ~(u64)0)
  780. return -1;
  781. vcpu->arch.mcg_ctl = data;
  782. break;
  783. default:
  784. if (msr >= MSR_IA32_MC0_CTL &&
  785. msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
  786. u32 offset = msr - MSR_IA32_MC0_CTL;
  787. /* only 0 or all 1s can be written to IA32_MCi_CTL */
  788. if ((offset & 0x3) == 0 &&
  789. data != 0 && data != ~(u64)0)
  790. return -1;
  791. vcpu->arch.mce_banks[offset] = data;
  792. break;
  793. }
  794. return 1;
  795. }
  796. return 0;
  797. }
  798. static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
  799. {
  800. struct kvm *kvm = vcpu->kvm;
  801. int lm = is_long_mode(vcpu);
  802. u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
  803. : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
  804. u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
  805. : kvm->arch.xen_hvm_config.blob_size_32;
  806. u32 page_num = data & ~PAGE_MASK;
  807. u64 page_addr = data & PAGE_MASK;
  808. u8 *page;
  809. int r;
  810. r = -E2BIG;
  811. if (page_num >= blob_size)
  812. goto out;
  813. r = -ENOMEM;
  814. page = kzalloc(PAGE_SIZE, GFP_KERNEL);
  815. if (!page)
  816. goto out;
  817. r = -EFAULT;
  818. if (copy_from_user(page, blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE))
  819. goto out_free;
  820. if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
  821. goto out_free;
  822. r = 0;
  823. out_free:
  824. kfree(page);
  825. out:
  826. return r;
  827. }
  828. int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  829. {
  830. switch (msr) {
  831. case MSR_EFER:
  832. set_efer(vcpu, data);
  833. break;
  834. case MSR_K7_HWCR:
  835. data &= ~(u64)0x40; /* ignore flush filter disable */
  836. if (data != 0) {
  837. pr_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
  838. data);
  839. return 1;
  840. }
  841. break;
  842. case MSR_FAM10H_MMIO_CONF_BASE:
  843. if (data != 0) {
  844. pr_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
  845. "0x%llx\n", data);
  846. return 1;
  847. }
  848. break;
  849. case MSR_AMD64_NB_CFG:
  850. break;
  851. case MSR_IA32_DEBUGCTLMSR:
  852. if (!data) {
  853. /* We support the non-activated case already */
  854. break;
  855. } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
  856. /* Values other than LBR and BTF are vendor-specific,
  857. thus reserved and should throw a #GP */
  858. return 1;
  859. }
  860. pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
  861. __func__, data);
  862. break;
  863. case MSR_IA32_UCODE_REV:
  864. case MSR_IA32_UCODE_WRITE:
  865. case MSR_VM_HSAVE_PA:
  866. case MSR_AMD64_PATCH_LOADER:
  867. break;
  868. case 0x200 ... 0x2ff:
  869. return set_msr_mtrr(vcpu, msr, data);
  870. case MSR_IA32_APICBASE:
  871. kvm_set_apic_base(vcpu, data);
  872. break;
  873. case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
  874. return kvm_x2apic_msr_write(vcpu, msr, data);
  875. case MSR_IA32_MISC_ENABLE:
  876. vcpu->arch.ia32_misc_enable_msr = data;
  877. break;
  878. case MSR_KVM_WALL_CLOCK:
  879. vcpu->kvm->arch.wall_clock = data;
  880. kvm_write_wall_clock(vcpu->kvm, data);
  881. break;
  882. case MSR_KVM_SYSTEM_TIME: {
  883. if (vcpu->arch.time_page) {
  884. kvm_release_page_dirty(vcpu->arch.time_page);
  885. vcpu->arch.time_page = NULL;
  886. }
  887. vcpu->arch.time = data;
  888. /* we verify if the enable bit is set... */
  889. if (!(data & 1))
  890. break;
  891. /* ...but clean it before doing the actual write */
  892. vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);
  893. vcpu->arch.time_page =
  894. gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);
  895. if (is_error_page(vcpu->arch.time_page)) {
  896. kvm_release_page_clean(vcpu->arch.time_page);
  897. vcpu->arch.time_page = NULL;
  898. }
  899. kvm_request_guest_time_update(vcpu);
  900. break;
  901. }
  902. case MSR_IA32_MCG_CTL:
  903. case MSR_IA32_MCG_STATUS:
  904. case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
  905. return set_msr_mce(vcpu, msr, data);
  906. /* Performance counters are not protected by a CPUID bit,
  907. * so we should check all of them in the generic path for the sake of
  908. * cross vendor migration.
  909. * Writing a zero into the event select MSRs disables them,
  910. * which we perfectly emulate ;-). Any other value should be at least
  911. * reported, some guests depend on them.
  912. */
  913. case MSR_P6_EVNTSEL0:
  914. case MSR_P6_EVNTSEL1:
  915. case MSR_K7_EVNTSEL0:
  916. case MSR_K7_EVNTSEL1:
  917. case MSR_K7_EVNTSEL2:
  918. case MSR_K7_EVNTSEL3:
  919. if (data != 0)
  920. pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
  921. "0x%x data 0x%llx\n", msr, data);
  922. break;
  923. /* at least RHEL 4 unconditionally writes to the perfctr registers,
  924. * so we ignore writes to make it happy.
  925. */
  926. case MSR_P6_PERFCTR0:
  927. case MSR_P6_PERFCTR1:
  928. case MSR_K7_PERFCTR0:
  929. case MSR_K7_PERFCTR1:
  930. case MSR_K7_PERFCTR2:
  931. case MSR_K7_PERFCTR3:
  932. pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
  933. "0x%x data 0x%llx\n", msr, data);
  934. break;
  935. default:
  936. if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
  937. return xen_hvm_config(vcpu, data);
  938. if (!ignore_msrs) {
  939. pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
  940. msr, data);
  941. return 1;
  942. } else {
  943. pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
  944. msr, data);
  945. break;
  946. }
  947. }
  948. return 0;
  949. }
  950. EXPORT_SYMBOL_GPL(kvm_set_msr_common);
  951. /*
  952. * Reads an msr value (of 'msr_index') into 'pdata'.
  953. * Returns 0 on success, non-0 otherwise.
  954. * Assumes vcpu_load() was already called.
  955. */
  956. int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
  957. {
  958. return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
  959. }
  960. static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  961. {
  962. u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
  963. if (!msr_mtrr_valid(msr))
  964. return 1;
  965. if (msr == MSR_MTRRdefType)
  966. *pdata = vcpu->arch.mtrr_state.def_type +
  967. (vcpu->arch.mtrr_state.enabled << 10);
  968. else if (msr == MSR_MTRRfix64K_00000)
  969. *pdata = p[0];
  970. else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
  971. *pdata = p[1 + msr - MSR_MTRRfix16K_80000];
  972. else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
  973. *pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
  974. else if (msr == MSR_IA32_CR_PAT)
  975. *pdata = vcpu->arch.pat;
  976. else { /* Variable MTRRs */
  977. int idx, is_mtrr_mask;
  978. u64 *pt;
  979. idx = (msr - 0x200) / 2;
  980. is_mtrr_mask = msr - 0x200 - 2 * idx;
  981. if (!is_mtrr_mask)
  982. pt =
  983. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
  984. else
  985. pt =
  986. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
  987. *pdata = *pt;
  988. }
  989. return 0;
  990. }
  991. static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  992. {
  993. u64 data;
  994. u64 mcg_cap = vcpu->arch.mcg_cap;
  995. unsigned bank_num = mcg_cap & 0xff;
  996. switch (msr) {
  997. case MSR_IA32_P5_MC_ADDR:
  998. case MSR_IA32_P5_MC_TYPE:
  999. data = 0;
  1000. break;
  1001. case MSR_IA32_MCG_CAP:
  1002. data = vcpu->arch.mcg_cap;
  1003. break;
  1004. case MSR_IA32_MCG_CTL:
  1005. if (!(mcg_cap & MCG_CTL_P))
  1006. return 1;
  1007. data = vcpu->arch.mcg_ctl;
  1008. break;
  1009. case MSR_IA32_MCG_STATUS:
  1010. data = vcpu->arch.mcg_status;
  1011. break;
  1012. default:
  1013. if (msr >= MSR_IA32_MC0_CTL &&
  1014. msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
  1015. u32 offset = msr - MSR_IA32_MC0_CTL;
  1016. data = vcpu->arch.mce_banks[offset];
  1017. break;
  1018. }
  1019. return 1;
  1020. }
  1021. *pdata = data;
  1022. return 0;
  1023. }
  1024. int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1025. {
  1026. u64 data;
  1027. switch (msr) {
  1028. case MSR_IA32_PLATFORM_ID:
  1029. case MSR_IA32_UCODE_REV:
  1030. case MSR_IA32_EBL_CR_POWERON:
  1031. case MSR_IA32_DEBUGCTLMSR:
  1032. case MSR_IA32_LASTBRANCHFROMIP:
  1033. case MSR_IA32_LASTBRANCHTOIP:
  1034. case MSR_IA32_LASTINTFROMIP:
  1035. case MSR_IA32_LASTINTTOIP:
  1036. case MSR_K8_SYSCFG:
  1037. case MSR_K7_HWCR:
  1038. case MSR_VM_HSAVE_PA:
  1039. case MSR_P6_PERFCTR0:
  1040. case MSR_P6_PERFCTR1:
  1041. case MSR_P6_EVNTSEL0:
  1042. case MSR_P6_EVNTSEL1:
  1043. case MSR_K7_EVNTSEL0:
  1044. case MSR_K7_PERFCTR0:
  1045. case MSR_K8_INT_PENDING_MSG:
  1046. case MSR_AMD64_NB_CFG:
  1047. case MSR_FAM10H_MMIO_CONF_BASE:
  1048. data = 0;
  1049. break;
  1050. case MSR_MTRRcap:
  1051. data = 0x500 | KVM_NR_VAR_MTRR;
  1052. break;
  1053. case 0x200 ... 0x2ff:
  1054. return get_msr_mtrr(vcpu, msr, pdata);
  1055. case 0xcd: /* fsb frequency */
  1056. data = 3;
  1057. break;
  1058. case MSR_IA32_APICBASE:
  1059. data = kvm_get_apic_base(vcpu);
  1060. break;
  1061. case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
  1062. return kvm_x2apic_msr_read(vcpu, msr, pdata);
  1063. break;
  1064. case MSR_IA32_MISC_ENABLE:
  1065. data = vcpu->arch.ia32_misc_enable_msr;
  1066. break;
  1067. case MSR_IA32_PERF_STATUS:
  1068. /* TSC increment by tick */
  1069. data = 1000ULL;
  1070. /* CPU multiplier */
  1071. data |= (((uint64_t)4ULL) << 40);
  1072. break;
  1073. case MSR_EFER:
  1074. data = vcpu->arch.shadow_efer;
  1075. break;
  1076. case MSR_KVM_WALL_CLOCK:
  1077. data = vcpu->kvm->arch.wall_clock;
  1078. break;
  1079. case MSR_KVM_SYSTEM_TIME:
  1080. data = vcpu->arch.time;
  1081. break;
  1082. case MSR_IA32_P5_MC_ADDR:
  1083. case MSR_IA32_P5_MC_TYPE:
  1084. case MSR_IA32_MCG_CAP:
  1085. case MSR_IA32_MCG_CTL:
  1086. case MSR_IA32_MCG_STATUS:
  1087. case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
  1088. return get_msr_mce(vcpu, msr, pdata);
  1089. default:
  1090. if (!ignore_msrs) {
  1091. pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
  1092. return 1;
  1093. } else {
  1094. pr_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
  1095. data = 0;
  1096. }
  1097. break;
  1098. }
  1099. *pdata = data;
  1100. return 0;
  1101. }
  1102. EXPORT_SYMBOL_GPL(kvm_get_msr_common);
  1103. /*
  1104. * Read or write a bunch of msrs. All parameters are kernel addresses.
  1105. *
  1106. * @return number of msrs set successfully.
  1107. */
  1108. static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
  1109. struct kvm_msr_entry *entries,
  1110. int (*do_msr)(struct kvm_vcpu *vcpu,
  1111. unsigned index, u64 *data))
  1112. {
  1113. int i;
  1114. vcpu_load(vcpu);
  1115. down_read(&vcpu->kvm->slots_lock);
  1116. for (i = 0; i < msrs->nmsrs; ++i)
  1117. if (do_msr(vcpu, entries[i].index, &entries[i].data))
  1118. break;
  1119. up_read(&vcpu->kvm->slots_lock);
  1120. vcpu_put(vcpu);
  1121. return i;
  1122. }
  1123. /*
  1124. * Read or write a bunch of msrs. Parameters are user addresses.
  1125. *
  1126. * @return number of msrs set successfully.
  1127. */
  1128. static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
  1129. int (*do_msr)(struct kvm_vcpu *vcpu,
  1130. unsigned index, u64 *data),
  1131. int writeback)
  1132. {
  1133. struct kvm_msrs msrs;
  1134. struct kvm_msr_entry *entries;
  1135. int r, n;
  1136. unsigned size;
  1137. r = -EFAULT;
  1138. if (copy_from_user(&msrs, user_msrs, sizeof msrs))
  1139. goto out;
  1140. r = -E2BIG;
  1141. if (msrs.nmsrs >= MAX_IO_MSRS)
  1142. goto out;
  1143. r = -ENOMEM;
  1144. size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
  1145. entries = vmalloc(size);
  1146. if (!entries)
  1147. goto out;
  1148. r = -EFAULT;
  1149. if (copy_from_user(entries, user_msrs->entries, size))
  1150. goto out_free;
  1151. r = n = __msr_io(vcpu, &msrs, entries, do_msr);
  1152. if (r < 0)
  1153. goto out_free;
  1154. r = -EFAULT;
  1155. if (writeback && copy_to_user(user_msrs->entries, entries, size))
  1156. goto out_free;
  1157. r = n;
  1158. out_free:
  1159. vfree(entries);
  1160. out:
  1161. return r;
  1162. }
  1163. int kvm_dev_ioctl_check_extension(long ext)
  1164. {
  1165. int r;
  1166. switch (ext) {
  1167. case KVM_CAP_IRQCHIP:
  1168. case KVM_CAP_HLT:
  1169. case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
  1170. case KVM_CAP_SET_TSS_ADDR:
  1171. case KVM_CAP_EXT_CPUID:
  1172. case KVM_CAP_CLOCKSOURCE:
  1173. case KVM_CAP_PIT:
  1174. case KVM_CAP_NOP_IO_DELAY:
  1175. case KVM_CAP_MP_STATE:
  1176. case KVM_CAP_SYNC_MMU:
  1177. case KVM_CAP_REINJECT_CONTROL:
  1178. case KVM_CAP_IRQ_INJECT_STATUS:
  1179. case KVM_CAP_ASSIGN_DEV_IRQ:
  1180. case KVM_CAP_IRQFD:
  1181. case KVM_CAP_IOEVENTFD:
  1182. case KVM_CAP_PIT2:
  1183. case KVM_CAP_PIT_STATE2:
  1184. case KVM_CAP_SET_IDENTITY_MAP_ADDR:
  1185. case KVM_CAP_XEN_HVM:
  1186. case KVM_CAP_ADJUST_CLOCK:
  1187. case KVM_CAP_VCPU_EVENTS:
  1188. r = 1;
  1189. break;
  1190. case KVM_CAP_COALESCED_MMIO:
  1191. r = KVM_COALESCED_MMIO_PAGE_OFFSET;
  1192. break;
  1193. case KVM_CAP_VAPIC:
  1194. r = !kvm_x86_ops->cpu_has_accelerated_tpr();
  1195. break;
  1196. case KVM_CAP_NR_VCPUS:
  1197. r = KVM_MAX_VCPUS;
  1198. break;
  1199. case KVM_CAP_NR_MEMSLOTS:
  1200. r = KVM_MEMORY_SLOTS;
  1201. break;
  1202. case KVM_CAP_PV_MMU: /* obsolete */
  1203. r = 0;
  1204. break;
  1205. case KVM_CAP_IOMMU:
  1206. r = iommu_found();
  1207. break;
  1208. case KVM_CAP_MCE:
  1209. r = KVM_MAX_MCE_BANKS;
  1210. break;
  1211. default:
  1212. r = 0;
  1213. break;
  1214. }
  1215. return r;
  1216. }
  1217. long kvm_arch_dev_ioctl(struct file *filp,
  1218. unsigned int ioctl, unsigned long arg)
  1219. {
  1220. void __user *argp = (void __user *)arg;
  1221. long r;
  1222. switch (ioctl) {
  1223. case KVM_GET_MSR_INDEX_LIST: {
  1224. struct kvm_msr_list __user *user_msr_list = argp;
  1225. struct kvm_msr_list msr_list;
  1226. unsigned n;
  1227. r = -EFAULT;
  1228. if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
  1229. goto out;
  1230. n = msr_list.nmsrs;
  1231. msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
  1232. if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
  1233. goto out;
  1234. r = -E2BIG;
  1235. if (n < msr_list.nmsrs)
  1236. goto out;
  1237. r = -EFAULT;
  1238. if (copy_to_user(user_msr_list->indices, &msrs_to_save,
  1239. num_msrs_to_save * sizeof(u32)))
  1240. goto out;
  1241. if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
  1242. &emulated_msrs,
  1243. ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
  1244. goto out;
  1245. r = 0;
  1246. break;
  1247. }
  1248. case KVM_GET_SUPPORTED_CPUID: {
  1249. struct kvm_cpuid2 __user *cpuid_arg = argp;
  1250. struct kvm_cpuid2 cpuid;
  1251. r = -EFAULT;
  1252. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1253. goto out;
  1254. r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
  1255. cpuid_arg->entries);
  1256. if (r)
  1257. goto out;
  1258. r = -EFAULT;
  1259. if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
  1260. goto out;
  1261. r = 0;
  1262. break;
  1263. }
  1264. case KVM_X86_GET_MCE_CAP_SUPPORTED: {
  1265. u64 mce_cap;
  1266. mce_cap = KVM_MCE_CAP_SUPPORTED;
  1267. r = -EFAULT;
  1268. if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
  1269. goto out;
  1270. r = 0;
  1271. break;
  1272. }
  1273. default:
  1274. r = -EINVAL;
  1275. }
  1276. out:
  1277. return r;
  1278. }
  1279. void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  1280. {
  1281. kvm_x86_ops->vcpu_load(vcpu, cpu);
  1282. if (unlikely(per_cpu(cpu_tsc_khz, cpu) == 0)) {
  1283. unsigned long khz = cpufreq_quick_get(cpu);
  1284. if (!khz)
  1285. khz = tsc_khz;
  1286. per_cpu(cpu_tsc_khz, cpu) = khz;
  1287. }
  1288. kvm_request_guest_time_update(vcpu);
  1289. }
  1290. void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
  1291. {
  1292. kvm_x86_ops->vcpu_put(vcpu);
  1293. kvm_put_guest_fpu(vcpu);
  1294. }
  1295. static int is_efer_nx(void)
  1296. {
  1297. unsigned long long efer = 0;
  1298. rdmsrl_safe(MSR_EFER, &efer);
  1299. return efer & EFER_NX;
  1300. }
  1301. static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
  1302. {
  1303. int i;
  1304. struct kvm_cpuid_entry2 *e, *entry;
  1305. entry = NULL;
  1306. for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
  1307. e = &vcpu->arch.cpuid_entries[i];
  1308. if (e->function == 0x80000001) {
  1309. entry = e;
  1310. break;
  1311. }
  1312. }
  1313. if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) {
  1314. entry->edx &= ~(1 << 20);
  1315. printk(KERN_INFO "kvm: guest NX capability removed\n");
  1316. }
  1317. }
  1318. /* when an old userspace process fills a new kernel module */
  1319. static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
  1320. struct kvm_cpuid *cpuid,
  1321. struct kvm_cpuid_entry __user *entries)
  1322. {
  1323. int r, i;
  1324. struct kvm_cpuid_entry *cpuid_entries;
  1325. r = -E2BIG;
  1326. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  1327. goto out;
  1328. r = -ENOMEM;
  1329. cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent);
  1330. if (!cpuid_entries)
  1331. goto out;
  1332. r = -EFAULT;
  1333. if (copy_from_user(cpuid_entries, entries,
  1334. cpuid->nent * sizeof(struct kvm_cpuid_entry)))
  1335. goto out_free;
  1336. for (i = 0; i < cpuid->nent; i++) {
  1337. vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
  1338. vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
  1339. vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
  1340. vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
  1341. vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
  1342. vcpu->arch.cpuid_entries[i].index = 0;
  1343. vcpu->arch.cpuid_entries[i].flags = 0;
  1344. vcpu->arch.cpuid_entries[i].padding[0] = 0;
  1345. vcpu->arch.cpuid_entries[i].padding[1] = 0;
  1346. vcpu->arch.cpuid_entries[i].padding[2] = 0;
  1347. }
  1348. vcpu->arch.cpuid_nent = cpuid->nent;
  1349. cpuid_fix_nx_cap(vcpu);
  1350. r = 0;
  1351. kvm_apic_set_version(vcpu);
  1352. out_free:
  1353. vfree(cpuid_entries);
  1354. out:
  1355. return r;
  1356. }
  1357. static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
  1358. struct kvm_cpuid2 *cpuid,
  1359. struct kvm_cpuid_entry2 __user *entries)
  1360. {
  1361. int r;
  1362. r = -E2BIG;
  1363. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  1364. goto out;
  1365. r = -EFAULT;
  1366. if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
  1367. cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
  1368. goto out;
  1369. vcpu->arch.cpuid_nent = cpuid->nent;
  1370. kvm_apic_set_version(vcpu);
  1371. return 0;
  1372. out:
  1373. return r;
  1374. }
  1375. static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
  1376. struct kvm_cpuid2 *cpuid,
  1377. struct kvm_cpuid_entry2 __user *entries)
  1378. {
  1379. int r;
  1380. r = -E2BIG;
  1381. if (cpuid->nent < vcpu->arch.cpuid_nent)
  1382. goto out;
  1383. r = -EFAULT;
  1384. if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
  1385. vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
  1386. goto out;
  1387. return 0;
  1388. out:
  1389. cpuid->nent = vcpu->arch.cpuid_nent;
  1390. return r;
  1391. }
  1392. static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
  1393. u32 index)
  1394. {
  1395. entry->function = function;
  1396. entry->index = index;
  1397. cpuid_count(entry->function, entry->index,
  1398. &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
  1399. entry->flags = 0;
  1400. }
  1401. #define F(x) bit(X86_FEATURE_##x)
  1402. static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
  1403. u32 index, int *nent, int maxnent)
  1404. {
  1405. unsigned f_nx = is_efer_nx() ? F(NX) : 0;
  1406. unsigned f_gbpages = kvm_x86_ops->gb_page_enable() ? F(GBPAGES) : 0;
  1407. #ifdef CONFIG_X86_64
  1408. unsigned f_lm = F(LM);
  1409. #else
  1410. unsigned f_lm = 0;
  1411. #endif
  1412. /* cpuid 1.edx */
  1413. const u32 kvm_supported_word0_x86_features =
  1414. F(FPU) | F(VME) | F(DE) | F(PSE) |
  1415. F(TSC) | F(MSR) | F(PAE) | F(MCE) |
  1416. F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
  1417. F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
  1418. F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLSH) |
  1419. 0 /* Reserved, DS, ACPI */ | F(MMX) |
  1420. F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
  1421. 0 /* HTT, TM, Reserved, PBE */;
  1422. /* cpuid 0x80000001.edx */
  1423. const u32 kvm_supported_word1_x86_features =
  1424. F(FPU) | F(VME) | F(DE) | F(PSE) |
  1425. F(TSC) | F(MSR) | F(PAE) | F(MCE) |
  1426. F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
  1427. F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
  1428. F(PAT) | F(PSE36) | 0 /* Reserved */ |
  1429. f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
  1430. F(FXSR) | F(FXSR_OPT) | f_gbpages | 0 /* RDTSCP */ |
  1431. 0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW);
  1432. /* cpuid 1.ecx */
  1433. const u32 kvm_supported_word4_x86_features =
  1434. F(XMM3) | 0 /* Reserved, DTES64, MONITOR */ |
  1435. 0 /* DS-CPL, VMX, SMX, EST */ |
  1436. 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
  1437. 0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ |
  1438. 0 /* Reserved, DCA */ | F(XMM4_1) |
  1439. F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
  1440. 0 /* Reserved, XSAVE, OSXSAVE */;
  1441. /* cpuid 0x80000001.ecx */
  1442. const u32 kvm_supported_word6_x86_features =
  1443. F(LAHF_LM) | F(CMP_LEGACY) | F(SVM) | 0 /* ExtApicSpace */ |
  1444. F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
  1445. F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(SSE5) |
  1446. 0 /* SKINIT */ | 0 /* WDT */;
  1447. /* all calls to cpuid_count() should be made on the same cpu */
  1448. get_cpu();
  1449. do_cpuid_1_ent(entry, function, index);
  1450. ++*nent;
  1451. switch (function) {
  1452. case 0:
  1453. entry->eax = min(entry->eax, (u32)0xb);
  1454. break;
  1455. case 1:
  1456. entry->edx &= kvm_supported_word0_x86_features;
  1457. entry->ecx &= kvm_supported_word4_x86_features;
  1458. /* we support x2apic emulation even if host does not support
  1459. * it since we emulate x2apic in software */
  1460. entry->ecx |= F(X2APIC);
  1461. break;
  1462. /* function 2 entries are STATEFUL. That is, repeated cpuid commands
  1463. * may return different values. This forces us to get_cpu() before
  1464. * issuing the first command, and also to emulate this annoying behavior
  1465. * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
  1466. case 2: {
  1467. int t, times = entry->eax & 0xff;
  1468. entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
  1469. entry->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
  1470. for (t = 1; t < times && *nent < maxnent; ++t) {
  1471. do_cpuid_1_ent(&entry[t], function, 0);
  1472. entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
  1473. ++*nent;
  1474. }
  1475. break;
  1476. }
  1477. /* function 4 and 0xb have additional index. */
  1478. case 4: {
  1479. int i, cache_type;
  1480. entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1481. /* read more entries until cache_type is zero */
  1482. for (i = 1; *nent < maxnent; ++i) {
  1483. cache_type = entry[i - 1].eax & 0x1f;
  1484. if (!cache_type)
  1485. break;
  1486. do_cpuid_1_ent(&entry[i], function, i);
  1487. entry[i].flags |=
  1488. KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1489. ++*nent;
  1490. }
  1491. break;
  1492. }
  1493. case 0xb: {
  1494. int i, level_type;
  1495. entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1496. /* read more entries until level_type is zero */
  1497. for (i = 1; *nent < maxnent; ++i) {
  1498. level_type = entry[i - 1].ecx & 0xff00;
  1499. if (!level_type)
  1500. break;
  1501. do_cpuid_1_ent(&entry[i], function, i);
  1502. entry[i].flags |=
  1503. KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1504. ++*nent;
  1505. }
  1506. break;
  1507. }
  1508. case 0x80000000:
  1509. entry->eax = min(entry->eax, 0x8000001a);
  1510. break;
  1511. case 0x80000001:
  1512. entry->edx &= kvm_supported_word1_x86_features;
  1513. entry->ecx &= kvm_supported_word6_x86_features;
  1514. break;
  1515. }
  1516. put_cpu();
  1517. }
  1518. #undef F
  1519. static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  1520. struct kvm_cpuid_entry2 __user *entries)
  1521. {
  1522. struct kvm_cpuid_entry2 *cpuid_entries;
  1523. int limit, nent = 0, r = -E2BIG;
  1524. u32 func;
  1525. if (cpuid->nent < 1)
  1526. goto out;
  1527. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  1528. cpuid->nent = KVM_MAX_CPUID_ENTRIES;
  1529. r = -ENOMEM;
  1530. cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent);
  1531. if (!cpuid_entries)
  1532. goto out;
  1533. do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent);
  1534. limit = cpuid_entries[0].eax;
  1535. for (func = 1; func <= limit && nent < cpuid->nent; ++func)
  1536. do_cpuid_ent(&cpuid_entries[nent], func, 0,
  1537. &nent, cpuid->nent);
  1538. r = -E2BIG;
  1539. if (nent >= cpuid->nent)
  1540. goto out_free;
  1541. do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent);
  1542. limit = cpuid_entries[nent - 1].eax;
  1543. for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func)
  1544. do_cpuid_ent(&cpuid_entries[nent], func, 0,
  1545. &nent, cpuid->nent);
  1546. r = -E2BIG;
  1547. if (nent >= cpuid->nent)
  1548. goto out_free;
  1549. r = -EFAULT;
  1550. if (copy_to_user(entries, cpuid_entries,
  1551. nent * sizeof(struct kvm_cpuid_entry2)))
  1552. goto out_free;
  1553. cpuid->nent = nent;
  1554. r = 0;
  1555. out_free:
  1556. vfree(cpuid_entries);
  1557. out:
  1558. return r;
  1559. }
  1560. static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
  1561. struct kvm_lapic_state *s)
  1562. {
  1563. vcpu_load(vcpu);
  1564. memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
  1565. vcpu_put(vcpu);
  1566. return 0;
  1567. }
  1568. static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
  1569. struct kvm_lapic_state *s)
  1570. {
  1571. vcpu_load(vcpu);
  1572. memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
  1573. kvm_apic_post_state_restore(vcpu);
  1574. update_cr8_intercept(vcpu);
  1575. vcpu_put(vcpu);
  1576. return 0;
  1577. }
  1578. static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
  1579. struct kvm_interrupt *irq)
  1580. {
  1581. if (irq->irq < 0 || irq->irq >= 256)
  1582. return -EINVAL;
  1583. if (irqchip_in_kernel(vcpu->kvm))
  1584. return -ENXIO;
  1585. vcpu_load(vcpu);
  1586. kvm_queue_interrupt(vcpu, irq->irq, false);
  1587. vcpu_put(vcpu);
  1588. return 0;
  1589. }
  1590. static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
  1591. {
  1592. vcpu_load(vcpu);
  1593. kvm_inject_nmi(vcpu);
  1594. vcpu_put(vcpu);
  1595. return 0;
  1596. }
  1597. static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
  1598. struct kvm_tpr_access_ctl *tac)
  1599. {
  1600. if (tac->flags)
  1601. return -EINVAL;
  1602. vcpu->arch.tpr_access_reporting = !!tac->enabled;
  1603. return 0;
  1604. }
  1605. static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
  1606. u64 mcg_cap)
  1607. {
  1608. int r;
  1609. unsigned bank_num = mcg_cap & 0xff, bank;
  1610. r = -EINVAL;
  1611. if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
  1612. goto out;
  1613. if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
  1614. goto out;
  1615. r = 0;
  1616. vcpu->arch.mcg_cap = mcg_cap;
  1617. /* Init IA32_MCG_CTL to all 1s */
  1618. if (mcg_cap & MCG_CTL_P)
  1619. vcpu->arch.mcg_ctl = ~(u64)0;
  1620. /* Init IA32_MCi_CTL to all 1s */
  1621. for (bank = 0; bank < bank_num; bank++)
  1622. vcpu->arch.mce_banks[bank*4] = ~(u64)0;
  1623. out:
  1624. return r;
  1625. }
  1626. static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
  1627. struct kvm_x86_mce *mce)
  1628. {
  1629. u64 mcg_cap = vcpu->arch.mcg_cap;
  1630. unsigned bank_num = mcg_cap & 0xff;
  1631. u64 *banks = vcpu->arch.mce_banks;
  1632. if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
  1633. return -EINVAL;
  1634. /*
  1635. * if IA32_MCG_CTL is not all 1s, the uncorrected error
  1636. * reporting is disabled
  1637. */
  1638. if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
  1639. vcpu->arch.mcg_ctl != ~(u64)0)
  1640. return 0;
  1641. banks += 4 * mce->bank;
  1642. /*
  1643. * if IA32_MCi_CTL is not all 1s, the uncorrected error
  1644. * reporting is disabled for the bank
  1645. */
  1646. if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
  1647. return 0;
  1648. if (mce->status & MCI_STATUS_UC) {
  1649. if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
  1650. !(vcpu->arch.cr4 & X86_CR4_MCE)) {
  1651. printk(KERN_DEBUG "kvm: set_mce: "
  1652. "injects mce exception while "
  1653. "previous one is in progress!\n");
  1654. set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
  1655. return 0;
  1656. }
  1657. if (banks[1] & MCI_STATUS_VAL)
  1658. mce->status |= MCI_STATUS_OVER;
  1659. banks[2] = mce->addr;
  1660. banks[3] = mce->misc;
  1661. vcpu->arch.mcg_status = mce->mcg_status;
  1662. banks[1] = mce->status;
  1663. kvm_queue_exception(vcpu, MC_VECTOR);
  1664. } else if (!(banks[1] & MCI_STATUS_VAL)
  1665. || !(banks[1] & MCI_STATUS_UC)) {
  1666. if (banks[1] & MCI_STATUS_VAL)
  1667. mce->status |= MCI_STATUS_OVER;
  1668. banks[2] = mce->addr;
  1669. banks[3] = mce->misc;
  1670. banks[1] = mce->status;
  1671. } else
  1672. banks[1] |= MCI_STATUS_OVER;
  1673. return 0;
  1674. }
  1675. static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
  1676. struct kvm_vcpu_events *events)
  1677. {
  1678. vcpu_load(vcpu);
  1679. events->exception.injected = vcpu->arch.exception.pending;
  1680. events->exception.nr = vcpu->arch.exception.nr;
  1681. events->exception.has_error_code = vcpu->arch.exception.has_error_code;
  1682. events->exception.error_code = vcpu->arch.exception.error_code;
  1683. events->interrupt.injected = vcpu->arch.interrupt.pending;
  1684. events->interrupt.nr = vcpu->arch.interrupt.nr;
  1685. events->interrupt.soft = vcpu->arch.interrupt.soft;
  1686. events->nmi.injected = vcpu->arch.nmi_injected;
  1687. events->nmi.pending = vcpu->arch.nmi_pending;
  1688. events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
  1689. events->sipi_vector = vcpu->arch.sipi_vector;
  1690. events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
  1691. | KVM_VCPUEVENT_VALID_SIPI_VECTOR);
  1692. vcpu_put(vcpu);
  1693. }
  1694. static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
  1695. struct kvm_vcpu_events *events)
  1696. {
  1697. if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
  1698. | KVM_VCPUEVENT_VALID_SIPI_VECTOR))
  1699. return -EINVAL;
  1700. vcpu_load(vcpu);
  1701. vcpu->arch.exception.pending = events->exception.injected;
  1702. vcpu->arch.exception.nr = events->exception.nr;
  1703. vcpu->arch.exception.has_error_code = events->exception.has_error_code;
  1704. vcpu->arch.exception.error_code = events->exception.error_code;
  1705. vcpu->arch.interrupt.pending = events->interrupt.injected;
  1706. vcpu->arch.interrupt.nr = events->interrupt.nr;
  1707. vcpu->arch.interrupt.soft = events->interrupt.soft;
  1708. if (vcpu->arch.interrupt.pending && irqchip_in_kernel(vcpu->kvm))
  1709. kvm_pic_clear_isr_ack(vcpu->kvm);
  1710. vcpu->arch.nmi_injected = events->nmi.injected;
  1711. if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
  1712. vcpu->arch.nmi_pending = events->nmi.pending;
  1713. kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);
  1714. if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR)
  1715. vcpu->arch.sipi_vector = events->sipi_vector;
  1716. vcpu_put(vcpu);
  1717. return 0;
  1718. }
  1719. long kvm_arch_vcpu_ioctl(struct file *filp,
  1720. unsigned int ioctl, unsigned long arg)
  1721. {
  1722. struct kvm_vcpu *vcpu = filp->private_data;
  1723. void __user *argp = (void __user *)arg;
  1724. int r;
  1725. struct kvm_lapic_state *lapic = NULL;
  1726. switch (ioctl) {
  1727. case KVM_GET_LAPIC: {
  1728. r = -EINVAL;
  1729. if (!vcpu->arch.apic)
  1730. goto out;
  1731. lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
  1732. r = -ENOMEM;
  1733. if (!lapic)
  1734. goto out;
  1735. r = kvm_vcpu_ioctl_get_lapic(vcpu, lapic);
  1736. if (r)
  1737. goto out;
  1738. r = -EFAULT;
  1739. if (copy_to_user(argp, lapic, sizeof(struct kvm_lapic_state)))
  1740. goto out;
  1741. r = 0;
  1742. break;
  1743. }
  1744. case KVM_SET_LAPIC: {
  1745. r = -EINVAL;
  1746. if (!vcpu->arch.apic)
  1747. goto out;
  1748. lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
  1749. r = -ENOMEM;
  1750. if (!lapic)
  1751. goto out;
  1752. r = -EFAULT;
  1753. if (copy_from_user(lapic, argp, sizeof(struct kvm_lapic_state)))
  1754. goto out;
  1755. r = kvm_vcpu_ioctl_set_lapic(vcpu, lapic);
  1756. if (r)
  1757. goto out;
  1758. r = 0;
  1759. break;
  1760. }
  1761. case KVM_INTERRUPT: {
  1762. struct kvm_interrupt irq;
  1763. r = -EFAULT;
  1764. if (copy_from_user(&irq, argp, sizeof irq))
  1765. goto out;
  1766. r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
  1767. if (r)
  1768. goto out;
  1769. r = 0;
  1770. break;
  1771. }
  1772. case KVM_NMI: {
  1773. r = kvm_vcpu_ioctl_nmi(vcpu);
  1774. if (r)
  1775. goto out;
  1776. r = 0;
  1777. break;
  1778. }
  1779. case KVM_SET_CPUID: {
  1780. struct kvm_cpuid __user *cpuid_arg = argp;
  1781. struct kvm_cpuid cpuid;
  1782. r = -EFAULT;
  1783. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1784. goto out;
  1785. r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
  1786. if (r)
  1787. goto out;
  1788. break;
  1789. }
  1790. case KVM_SET_CPUID2: {
  1791. struct kvm_cpuid2 __user *cpuid_arg = argp;
  1792. struct kvm_cpuid2 cpuid;
  1793. r = -EFAULT;
  1794. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1795. goto out;
  1796. r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
  1797. cpuid_arg->entries);
  1798. if (r)
  1799. goto out;
  1800. break;
  1801. }
  1802. case KVM_GET_CPUID2: {
  1803. struct kvm_cpuid2 __user *cpuid_arg = argp;
  1804. struct kvm_cpuid2 cpuid;
  1805. r = -EFAULT;
  1806. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1807. goto out;
  1808. r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
  1809. cpuid_arg->entries);
  1810. if (r)
  1811. goto out;
  1812. r = -EFAULT;
  1813. if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
  1814. goto out;
  1815. r = 0;
  1816. break;
  1817. }
  1818. case KVM_GET_MSRS:
  1819. r = msr_io(vcpu, argp, kvm_get_msr, 1);
  1820. break;
  1821. case KVM_SET_MSRS:
  1822. r = msr_io(vcpu, argp, do_set_msr, 0);
  1823. break;
  1824. case KVM_TPR_ACCESS_REPORTING: {
  1825. struct kvm_tpr_access_ctl tac;
  1826. r = -EFAULT;
  1827. if (copy_from_user(&tac, argp, sizeof tac))
  1828. goto out;
  1829. r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
  1830. if (r)
  1831. goto out;
  1832. r = -EFAULT;
  1833. if (copy_to_user(argp, &tac, sizeof tac))
  1834. goto out;
  1835. r = 0;
  1836. break;
  1837. };
  1838. case KVM_SET_VAPIC_ADDR: {
  1839. struct kvm_vapic_addr va;
  1840. r = -EINVAL;
  1841. if (!irqchip_in_kernel(vcpu->kvm))
  1842. goto out;
  1843. r = -EFAULT;
  1844. if (copy_from_user(&va, argp, sizeof va))
  1845. goto out;
  1846. r = 0;
  1847. kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
  1848. break;
  1849. }
  1850. case KVM_X86_SETUP_MCE: {
  1851. u64 mcg_cap;
  1852. r = -EFAULT;
  1853. if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
  1854. goto out;
  1855. r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
  1856. break;
  1857. }
  1858. case KVM_X86_SET_MCE: {
  1859. struct kvm_x86_mce mce;
  1860. r = -EFAULT;
  1861. if (copy_from_user(&mce, argp, sizeof mce))
  1862. goto out;
  1863. r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
  1864. break;
  1865. }
  1866. case KVM_GET_VCPU_EVENTS: {
  1867. struct kvm_vcpu_events events;
  1868. kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
  1869. r = -EFAULT;
  1870. if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
  1871. break;
  1872. r = 0;
  1873. break;
  1874. }
  1875. case KVM_SET_VCPU_EVENTS: {
  1876. struct kvm_vcpu_events events;
  1877. r = -EFAULT;
  1878. if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
  1879. break;
  1880. r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
  1881. break;
  1882. }
  1883. default:
  1884. r = -EINVAL;
  1885. }
  1886. out:
  1887. kfree(lapic);
  1888. return r;
  1889. }
  1890. static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
  1891. {
  1892. int ret;
  1893. if (addr > (unsigned int)(-3 * PAGE_SIZE))
  1894. return -1;
  1895. ret = kvm_x86_ops->set_tss_addr(kvm, addr);
  1896. return ret;
  1897. }
  1898. static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
  1899. u64 ident_addr)
  1900. {
  1901. kvm->arch.ept_identity_map_addr = ident_addr;
  1902. return 0;
  1903. }
  1904. static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
  1905. u32 kvm_nr_mmu_pages)
  1906. {
  1907. if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
  1908. return -EINVAL;
  1909. down_write(&kvm->slots_lock);
  1910. spin_lock(&kvm->mmu_lock);
  1911. kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
  1912. kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
  1913. spin_unlock(&kvm->mmu_lock);
  1914. up_write(&kvm->slots_lock);
  1915. return 0;
  1916. }
  1917. static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
  1918. {
  1919. return kvm->arch.n_alloc_mmu_pages;
  1920. }
  1921. gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
  1922. {
  1923. int i;
  1924. struct kvm_mem_alias *alias;
  1925. for (i = 0; i < kvm->arch.naliases; ++i) {
  1926. alias = &kvm->arch.aliases[i];
  1927. if (gfn >= alias->base_gfn
  1928. && gfn < alias->base_gfn + alias->npages)
  1929. return alias->target_gfn + gfn - alias->base_gfn;
  1930. }
  1931. return gfn;
  1932. }
  1933. /*
  1934. * Set a new alias region. Aliases map a portion of physical memory into
  1935. * another portion. This is useful for memory windows, for example the PC
  1936. * VGA region.
  1937. */
  1938. static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
  1939. struct kvm_memory_alias *alias)
  1940. {
  1941. int r, n;
  1942. struct kvm_mem_alias *p;
  1943. r = -EINVAL;
  1944. /* General sanity checks */
  1945. if (alias->memory_size & (PAGE_SIZE - 1))
  1946. goto out;
  1947. if (alias->guest_phys_addr & (PAGE_SIZE - 1))
  1948. goto out;
  1949. if (alias->slot >= KVM_ALIAS_SLOTS)
  1950. goto out;
  1951. if (alias->guest_phys_addr + alias->memory_size
  1952. < alias->guest_phys_addr)
  1953. goto out;
  1954. if (alias->target_phys_addr + alias->memory_size
  1955. < alias->target_phys_addr)
  1956. goto out;
  1957. down_write(&kvm->slots_lock);
  1958. spin_lock(&kvm->mmu_lock);
  1959. p = &kvm->arch.aliases[alias->slot];
  1960. p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
  1961. p->npages = alias->memory_size >> PAGE_SHIFT;
  1962. p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
  1963. for (n = KVM_ALIAS_SLOTS; n > 0; --n)
  1964. if (kvm->arch.aliases[n - 1].npages)
  1965. break;
  1966. kvm->arch.naliases = n;
  1967. spin_unlock(&kvm->mmu_lock);
  1968. kvm_mmu_zap_all(kvm);
  1969. up_write(&kvm->slots_lock);
  1970. return 0;
  1971. out:
  1972. return r;
  1973. }
  1974. static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
  1975. {
  1976. int r;
  1977. r = 0;
  1978. switch (chip->chip_id) {
  1979. case KVM_IRQCHIP_PIC_MASTER:
  1980. memcpy(&chip->chip.pic,
  1981. &pic_irqchip(kvm)->pics[0],
  1982. sizeof(struct kvm_pic_state));
  1983. break;
  1984. case KVM_IRQCHIP_PIC_SLAVE:
  1985. memcpy(&chip->chip.pic,
  1986. &pic_irqchip(kvm)->pics[1],
  1987. sizeof(struct kvm_pic_state));
  1988. break;
  1989. case KVM_IRQCHIP_IOAPIC:
  1990. r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
  1991. break;
  1992. default:
  1993. r = -EINVAL;
  1994. break;
  1995. }
  1996. return r;
  1997. }
  1998. static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
  1999. {
  2000. int r;
  2001. r = 0;
  2002. switch (chip->chip_id) {
  2003. case KVM_IRQCHIP_PIC_MASTER:
  2004. spin_lock(&pic_irqchip(kvm)->lock);
  2005. memcpy(&pic_irqchip(kvm)->pics[0],
  2006. &chip->chip.pic,
  2007. sizeof(struct kvm_pic_state));
  2008. spin_unlock(&pic_irqchip(kvm)->lock);
  2009. break;
  2010. case KVM_IRQCHIP_PIC_SLAVE:
  2011. spin_lock(&pic_irqchip(kvm)->lock);
  2012. memcpy(&pic_irqchip(kvm)->pics[1],
  2013. &chip->chip.pic,
  2014. sizeof(struct kvm_pic_state));
  2015. spin_unlock(&pic_irqchip(kvm)->lock);
  2016. break;
  2017. case KVM_IRQCHIP_IOAPIC:
  2018. r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
  2019. break;
  2020. default:
  2021. r = -EINVAL;
  2022. break;
  2023. }
  2024. kvm_pic_update_irq(pic_irqchip(kvm));
  2025. return r;
  2026. }
  2027. static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
  2028. {
  2029. int r = 0;
  2030. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2031. memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
  2032. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2033. return r;
  2034. }
  2035. static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
  2036. {
  2037. int r = 0;
  2038. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2039. memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
  2040. kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
  2041. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2042. return r;
  2043. }
  2044. static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
  2045. {
  2046. int r = 0;
  2047. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2048. memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
  2049. sizeof(ps->channels));
  2050. ps->flags = kvm->arch.vpit->pit_state.flags;
  2051. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2052. return r;
  2053. }
  2054. static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
  2055. {
  2056. int r = 0, start = 0;
  2057. u32 prev_legacy, cur_legacy;
  2058. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2059. prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
  2060. cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
  2061. if (!prev_legacy && cur_legacy)
  2062. start = 1;
  2063. memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
  2064. sizeof(kvm->arch.vpit->pit_state.channels));
  2065. kvm->arch.vpit->pit_state.flags = ps->flags;
  2066. kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
  2067. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2068. return r;
  2069. }
  2070. static int kvm_vm_ioctl_reinject(struct kvm *kvm,
  2071. struct kvm_reinject_control *control)
  2072. {
  2073. if (!kvm->arch.vpit)
  2074. return -ENXIO;
  2075. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2076. kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject;
  2077. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2078. return 0;
  2079. }
  2080. /*
  2081. * Get (and clear) the dirty memory log for a memory slot.
  2082. */
  2083. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
  2084. struct kvm_dirty_log *log)
  2085. {
  2086. int r;
  2087. int n;
  2088. struct kvm_memory_slot *memslot;
  2089. int is_dirty = 0;
  2090. down_write(&kvm->slots_lock);
  2091. r = kvm_get_dirty_log(kvm, log, &is_dirty);
  2092. if (r)
  2093. goto out;
  2094. /* If nothing is dirty, don't bother messing with page tables. */
  2095. if (is_dirty) {
  2096. spin_lock(&kvm->mmu_lock);
  2097. kvm_mmu_slot_remove_write_access(kvm, log->slot);
  2098. spin_unlock(&kvm->mmu_lock);
  2099. memslot = &kvm->memslots[log->slot];
  2100. n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
  2101. memset(memslot->dirty_bitmap, 0, n);
  2102. }
  2103. r = 0;
  2104. out:
  2105. up_write(&kvm->slots_lock);
  2106. return r;
  2107. }
  2108. long kvm_arch_vm_ioctl(struct file *filp,
  2109. unsigned int ioctl, unsigned long arg)
  2110. {
  2111. struct kvm *kvm = filp->private_data;
  2112. void __user *argp = (void __user *)arg;
  2113. int r = -ENOTTY;
  2114. /*
  2115. * This union makes it completely explicit to gcc-3.x
  2116. * that these two variables' stack usage should be
  2117. * combined, not added together.
  2118. */
  2119. union {
  2120. struct kvm_pit_state ps;
  2121. struct kvm_pit_state2 ps2;
  2122. struct kvm_memory_alias alias;
  2123. struct kvm_pit_config pit_config;
  2124. } u;
  2125. switch (ioctl) {
  2126. case KVM_SET_TSS_ADDR:
  2127. r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
  2128. if (r < 0)
  2129. goto out;
  2130. break;
  2131. case KVM_SET_IDENTITY_MAP_ADDR: {
  2132. u64 ident_addr;
  2133. r = -EFAULT;
  2134. if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
  2135. goto out;
  2136. r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
  2137. if (r < 0)
  2138. goto out;
  2139. break;
  2140. }
  2141. case KVM_SET_MEMORY_REGION: {
  2142. struct kvm_memory_region kvm_mem;
  2143. struct kvm_userspace_memory_region kvm_userspace_mem;
  2144. r = -EFAULT;
  2145. if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
  2146. goto out;
  2147. kvm_userspace_mem.slot = kvm_mem.slot;
  2148. kvm_userspace_mem.flags = kvm_mem.flags;
  2149. kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr;
  2150. kvm_userspace_mem.memory_size = kvm_mem.memory_size;
  2151. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0);
  2152. if (r)
  2153. goto out;
  2154. break;
  2155. }
  2156. case KVM_SET_NR_MMU_PAGES:
  2157. r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
  2158. if (r)
  2159. goto out;
  2160. break;
  2161. case KVM_GET_NR_MMU_PAGES:
  2162. r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
  2163. break;
  2164. case KVM_SET_MEMORY_ALIAS:
  2165. r = -EFAULT;
  2166. if (copy_from_user(&u.alias, argp, sizeof(struct kvm_memory_alias)))
  2167. goto out;
  2168. r = kvm_vm_ioctl_set_memory_alias(kvm, &u.alias);
  2169. if (r)
  2170. goto out;
  2171. break;
  2172. case KVM_CREATE_IRQCHIP: {
  2173. struct kvm_pic *vpic;
  2174. mutex_lock(&kvm->lock);
  2175. r = -EEXIST;
  2176. if (kvm->arch.vpic)
  2177. goto create_irqchip_unlock;
  2178. r = -ENOMEM;
  2179. vpic = kvm_create_pic(kvm);
  2180. if (vpic) {
  2181. r = kvm_ioapic_init(kvm);
  2182. if (r) {
  2183. kfree(vpic);
  2184. goto create_irqchip_unlock;
  2185. }
  2186. } else
  2187. goto create_irqchip_unlock;
  2188. smp_wmb();
  2189. kvm->arch.vpic = vpic;
  2190. smp_wmb();
  2191. r = kvm_setup_default_irq_routing(kvm);
  2192. if (r) {
  2193. mutex_lock(&kvm->irq_lock);
  2194. kfree(kvm->arch.vpic);
  2195. kfree(kvm->arch.vioapic);
  2196. kvm->arch.vpic = NULL;
  2197. kvm->arch.vioapic = NULL;
  2198. mutex_unlock(&kvm->irq_lock);
  2199. }
  2200. create_irqchip_unlock:
  2201. mutex_unlock(&kvm->lock);
  2202. break;
  2203. }
  2204. case KVM_CREATE_PIT:
  2205. u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
  2206. goto create_pit;
  2207. case KVM_CREATE_PIT2:
  2208. r = -EFAULT;
  2209. if (copy_from_user(&u.pit_config, argp,
  2210. sizeof(struct kvm_pit_config)))
  2211. goto out;
  2212. create_pit:
  2213. down_write(&kvm->slots_lock);
  2214. r = -EEXIST;
  2215. if (kvm->arch.vpit)
  2216. goto create_pit_unlock;
  2217. r = -ENOMEM;
  2218. kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
  2219. if (kvm->arch.vpit)
  2220. r = 0;
  2221. create_pit_unlock:
  2222. up_write(&kvm->slots_lock);
  2223. break;
  2224. case KVM_IRQ_LINE_STATUS:
  2225. case KVM_IRQ_LINE: {
  2226. struct kvm_irq_level irq_event;
  2227. r = -EFAULT;
  2228. if (copy_from_user(&irq_event, argp, sizeof irq_event))
  2229. goto out;
  2230. if (irqchip_in_kernel(kvm)) {
  2231. __s32 status;
  2232. status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
  2233. irq_event.irq, irq_event.level);
  2234. if (ioctl == KVM_IRQ_LINE_STATUS) {
  2235. irq_event.status = status;
  2236. if (copy_to_user(argp, &irq_event,
  2237. sizeof irq_event))
  2238. goto out;
  2239. }
  2240. r = 0;
  2241. }
  2242. break;
  2243. }
  2244. case KVM_GET_IRQCHIP: {
  2245. /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
  2246. struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
  2247. r = -ENOMEM;
  2248. if (!chip)
  2249. goto out;
  2250. r = -EFAULT;
  2251. if (copy_from_user(chip, argp, sizeof *chip))
  2252. goto get_irqchip_out;
  2253. r = -ENXIO;
  2254. if (!irqchip_in_kernel(kvm))
  2255. goto get_irqchip_out;
  2256. r = kvm_vm_ioctl_get_irqchip(kvm, chip);
  2257. if (r)
  2258. goto get_irqchip_out;
  2259. r = -EFAULT;
  2260. if (copy_to_user(argp, chip, sizeof *chip))
  2261. goto get_irqchip_out;
  2262. r = 0;
  2263. get_irqchip_out:
  2264. kfree(chip);
  2265. if (r)
  2266. goto out;
  2267. break;
  2268. }
  2269. case KVM_SET_IRQCHIP: {
  2270. /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
  2271. struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
  2272. r = -ENOMEM;
  2273. if (!chip)
  2274. goto out;
  2275. r = -EFAULT;
  2276. if (copy_from_user(chip, argp, sizeof *chip))
  2277. goto set_irqchip_out;
  2278. r = -ENXIO;
  2279. if (!irqchip_in_kernel(kvm))
  2280. goto set_irqchip_out;
  2281. r = kvm_vm_ioctl_set_irqchip(kvm, chip);
  2282. if (r)
  2283. goto set_irqchip_out;
  2284. r = 0;
  2285. set_irqchip_out:
  2286. kfree(chip);
  2287. if (r)
  2288. goto out;
  2289. break;
  2290. }
  2291. case KVM_GET_PIT: {
  2292. r = -EFAULT;
  2293. if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
  2294. goto out;
  2295. r = -ENXIO;
  2296. if (!kvm->arch.vpit)
  2297. goto out;
  2298. r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
  2299. if (r)
  2300. goto out;
  2301. r = -EFAULT;
  2302. if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
  2303. goto out;
  2304. r = 0;
  2305. break;
  2306. }
  2307. case KVM_SET_PIT: {
  2308. r = -EFAULT;
  2309. if (copy_from_user(&u.ps, argp, sizeof u.ps))
  2310. goto out;
  2311. r = -ENXIO;
  2312. if (!kvm->arch.vpit)
  2313. goto out;
  2314. r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
  2315. if (r)
  2316. goto out;
  2317. r = 0;
  2318. break;
  2319. }
  2320. case KVM_GET_PIT2: {
  2321. r = -ENXIO;
  2322. if (!kvm->arch.vpit)
  2323. goto out;
  2324. r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
  2325. if (r)
  2326. goto out;
  2327. r = -EFAULT;
  2328. if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
  2329. goto out;
  2330. r = 0;
  2331. break;
  2332. }
  2333. case KVM_SET_PIT2: {
  2334. r = -EFAULT;
  2335. if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
  2336. goto out;
  2337. r = -ENXIO;
  2338. if (!kvm->arch.vpit)
  2339. goto out;
  2340. r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
  2341. if (r)
  2342. goto out;
  2343. r = 0;
  2344. break;
  2345. }
  2346. case KVM_REINJECT_CONTROL: {
  2347. struct kvm_reinject_control control;
  2348. r = -EFAULT;
  2349. if (copy_from_user(&control, argp, sizeof(control)))
  2350. goto out;
  2351. r = kvm_vm_ioctl_reinject(kvm, &control);
  2352. if (r)
  2353. goto out;
  2354. r = 0;
  2355. break;
  2356. }
  2357. case KVM_XEN_HVM_CONFIG: {
  2358. r = -EFAULT;
  2359. if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
  2360. sizeof(struct kvm_xen_hvm_config)))
  2361. goto out;
  2362. r = -EINVAL;
  2363. if (kvm->arch.xen_hvm_config.flags)
  2364. goto out;
  2365. r = 0;
  2366. break;
  2367. }
  2368. case KVM_SET_CLOCK: {
  2369. struct timespec now;
  2370. struct kvm_clock_data user_ns;
  2371. u64 now_ns;
  2372. s64 delta;
  2373. r = -EFAULT;
  2374. if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
  2375. goto out;
  2376. r = -EINVAL;
  2377. if (user_ns.flags)
  2378. goto out;
  2379. r = 0;
  2380. ktime_get_ts(&now);
  2381. now_ns = timespec_to_ns(&now);
  2382. delta = user_ns.clock - now_ns;
  2383. kvm->arch.kvmclock_offset = delta;
  2384. break;
  2385. }
  2386. case KVM_GET_CLOCK: {
  2387. struct timespec now;
  2388. struct kvm_clock_data user_ns;
  2389. u64 now_ns;
  2390. ktime_get_ts(&now);
  2391. now_ns = timespec_to_ns(&now);
  2392. user_ns.clock = kvm->arch.kvmclock_offset + now_ns;
  2393. user_ns.flags = 0;
  2394. r = -EFAULT;
  2395. if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
  2396. goto out;
  2397. r = 0;
  2398. break;
  2399. }
  2400. default:
  2401. ;
  2402. }
  2403. out:
  2404. return r;
  2405. }
  2406. static void kvm_init_msr_list(void)
  2407. {
  2408. u32 dummy[2];
  2409. unsigned i, j;
  2410. /* skip the first msrs in the list. KVM-specific */
  2411. for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) {
  2412. if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
  2413. continue;
  2414. if (j < i)
  2415. msrs_to_save[j] = msrs_to_save[i];
  2416. j++;
  2417. }
  2418. num_msrs_to_save = j;
  2419. }
  2420. static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
  2421. const void *v)
  2422. {
  2423. if (vcpu->arch.apic &&
  2424. !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, len, v))
  2425. return 0;
  2426. return kvm_io_bus_write(&vcpu->kvm->mmio_bus, addr, len, v);
  2427. }
  2428. static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
  2429. {
  2430. if (vcpu->arch.apic &&
  2431. !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, len, v))
  2432. return 0;
  2433. return kvm_io_bus_read(&vcpu->kvm->mmio_bus, addr, len, v);
  2434. }
  2435. static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes,
  2436. struct kvm_vcpu *vcpu)
  2437. {
  2438. void *data = val;
  2439. int r = X86EMUL_CONTINUE;
  2440. while (bytes) {
  2441. gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
  2442. unsigned offset = addr & (PAGE_SIZE-1);
  2443. unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
  2444. int ret;
  2445. if (gpa == UNMAPPED_GVA) {
  2446. r = X86EMUL_PROPAGATE_FAULT;
  2447. goto out;
  2448. }
  2449. ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
  2450. if (ret < 0) {
  2451. r = X86EMUL_UNHANDLEABLE;
  2452. goto out;
  2453. }
  2454. bytes -= toread;
  2455. data += toread;
  2456. addr += toread;
  2457. }
  2458. out:
  2459. return r;
  2460. }
  2461. static int kvm_write_guest_virt(gva_t addr, void *val, unsigned int bytes,
  2462. struct kvm_vcpu *vcpu)
  2463. {
  2464. void *data = val;
  2465. int r = X86EMUL_CONTINUE;
  2466. while (bytes) {
  2467. gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
  2468. unsigned offset = addr & (PAGE_SIZE-1);
  2469. unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
  2470. int ret;
  2471. if (gpa == UNMAPPED_GVA) {
  2472. r = X86EMUL_PROPAGATE_FAULT;
  2473. goto out;
  2474. }
  2475. ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
  2476. if (ret < 0) {
  2477. r = X86EMUL_UNHANDLEABLE;
  2478. goto out;
  2479. }
  2480. bytes -= towrite;
  2481. data += towrite;
  2482. addr += towrite;
  2483. }
  2484. out:
  2485. return r;
  2486. }
  2487. static int emulator_read_emulated(unsigned long addr,
  2488. void *val,
  2489. unsigned int bytes,
  2490. struct kvm_vcpu *vcpu)
  2491. {
  2492. gpa_t gpa;
  2493. if (vcpu->mmio_read_completed) {
  2494. memcpy(val, vcpu->mmio_data, bytes);
  2495. trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
  2496. vcpu->mmio_phys_addr, *(u64 *)val);
  2497. vcpu->mmio_read_completed = 0;
  2498. return X86EMUL_CONTINUE;
  2499. }
  2500. gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
  2501. /* For APIC access vmexit */
  2502. if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  2503. goto mmio;
  2504. if (kvm_read_guest_virt(addr, val, bytes, vcpu)
  2505. == X86EMUL_CONTINUE)
  2506. return X86EMUL_CONTINUE;
  2507. if (gpa == UNMAPPED_GVA)
  2508. return X86EMUL_PROPAGATE_FAULT;
  2509. mmio:
  2510. /*
  2511. * Is this MMIO handled locally?
  2512. */
  2513. if (!vcpu_mmio_read(vcpu, gpa, bytes, val)) {
  2514. trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes, gpa, *(u64 *)val);
  2515. return X86EMUL_CONTINUE;
  2516. }
  2517. trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
  2518. vcpu->mmio_needed = 1;
  2519. vcpu->mmio_phys_addr = gpa;
  2520. vcpu->mmio_size = bytes;
  2521. vcpu->mmio_is_write = 0;
  2522. return X86EMUL_UNHANDLEABLE;
  2523. }
  2524. int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
  2525. const void *val, int bytes)
  2526. {
  2527. int ret;
  2528. ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
  2529. if (ret < 0)
  2530. return 0;
  2531. kvm_mmu_pte_write(vcpu, gpa, val, bytes, 1);
  2532. return 1;
  2533. }
  2534. static int emulator_write_emulated_onepage(unsigned long addr,
  2535. const void *val,
  2536. unsigned int bytes,
  2537. struct kvm_vcpu *vcpu)
  2538. {
  2539. gpa_t gpa;
  2540. gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
  2541. if (gpa == UNMAPPED_GVA) {
  2542. kvm_inject_page_fault(vcpu, addr, 2);
  2543. return X86EMUL_PROPAGATE_FAULT;
  2544. }
  2545. /* For APIC access vmexit */
  2546. if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  2547. goto mmio;
  2548. if (emulator_write_phys(vcpu, gpa, val, bytes))
  2549. return X86EMUL_CONTINUE;
  2550. mmio:
  2551. trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
  2552. /*
  2553. * Is this MMIO handled locally?
  2554. */
  2555. if (!vcpu_mmio_write(vcpu, gpa, bytes, val))
  2556. return X86EMUL_CONTINUE;
  2557. vcpu->mmio_needed = 1;
  2558. vcpu->mmio_phys_addr = gpa;
  2559. vcpu->mmio_size = bytes;
  2560. vcpu->mmio_is_write = 1;
  2561. memcpy(vcpu->mmio_data, val, bytes);
  2562. return X86EMUL_CONTINUE;
  2563. }
  2564. int emulator_write_emulated(unsigned long addr,
  2565. const void *val,
  2566. unsigned int bytes,
  2567. struct kvm_vcpu *vcpu)
  2568. {
  2569. /* Crossing a page boundary? */
  2570. if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
  2571. int rc, now;
  2572. now = -addr & ~PAGE_MASK;
  2573. rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
  2574. if (rc != X86EMUL_CONTINUE)
  2575. return rc;
  2576. addr += now;
  2577. val += now;
  2578. bytes -= now;
  2579. }
  2580. return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
  2581. }
  2582. EXPORT_SYMBOL_GPL(emulator_write_emulated);
  2583. static int emulator_cmpxchg_emulated(unsigned long addr,
  2584. const void *old,
  2585. const void *new,
  2586. unsigned int bytes,
  2587. struct kvm_vcpu *vcpu)
  2588. {
  2589. printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
  2590. #ifndef CONFIG_X86_64
  2591. /* guests cmpxchg8b have to be emulated atomically */
  2592. if (bytes == 8) {
  2593. gpa_t gpa;
  2594. struct page *page;
  2595. char *kaddr;
  2596. u64 val;
  2597. gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
  2598. if (gpa == UNMAPPED_GVA ||
  2599. (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  2600. goto emul_write;
  2601. if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
  2602. goto emul_write;
  2603. val = *(u64 *)new;
  2604. page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  2605. kaddr = kmap_atomic(page, KM_USER0);
  2606. set_64bit((u64 *)(kaddr + offset_in_page(gpa)), val);
  2607. kunmap_atomic(kaddr, KM_USER0);
  2608. kvm_release_page_dirty(page);
  2609. }
  2610. emul_write:
  2611. #endif
  2612. return emulator_write_emulated(addr, new, bytes, vcpu);
  2613. }
  2614. static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
  2615. {
  2616. return kvm_x86_ops->get_segment_base(vcpu, seg);
  2617. }
  2618. int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
  2619. {
  2620. kvm_mmu_invlpg(vcpu, address);
  2621. return X86EMUL_CONTINUE;
  2622. }
  2623. int emulate_clts(struct kvm_vcpu *vcpu)
  2624. {
  2625. kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS);
  2626. return X86EMUL_CONTINUE;
  2627. }
  2628. int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
  2629. {
  2630. struct kvm_vcpu *vcpu = ctxt->vcpu;
  2631. switch (dr) {
  2632. case 0 ... 3:
  2633. *dest = kvm_x86_ops->get_dr(vcpu, dr);
  2634. return X86EMUL_CONTINUE;
  2635. default:
  2636. pr_unimpl(vcpu, "%s: unexpected dr %u\n", __func__, dr);
  2637. return X86EMUL_UNHANDLEABLE;
  2638. }
  2639. }
  2640. int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
  2641. {
  2642. unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
  2643. int exception;
  2644. kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
  2645. if (exception) {
  2646. /* FIXME: better handling */
  2647. return X86EMUL_UNHANDLEABLE;
  2648. }
  2649. return X86EMUL_CONTINUE;
  2650. }
  2651. void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context)
  2652. {
  2653. u8 opcodes[4];
  2654. unsigned long rip = kvm_rip_read(vcpu);
  2655. unsigned long rip_linear;
  2656. if (!printk_ratelimit())
  2657. return;
  2658. rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS);
  2659. kvm_read_guest_virt(rip_linear, (void *)opcodes, 4, vcpu);
  2660. printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n",
  2661. context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
  2662. }
  2663. EXPORT_SYMBOL_GPL(kvm_report_emulation_failure);
  2664. static struct x86_emulate_ops emulate_ops = {
  2665. .read_std = kvm_read_guest_virt,
  2666. .read_emulated = emulator_read_emulated,
  2667. .write_emulated = emulator_write_emulated,
  2668. .cmpxchg_emulated = emulator_cmpxchg_emulated,
  2669. };
  2670. static void cache_all_regs(struct kvm_vcpu *vcpu)
  2671. {
  2672. kvm_register_read(vcpu, VCPU_REGS_RAX);
  2673. kvm_register_read(vcpu, VCPU_REGS_RSP);
  2674. kvm_register_read(vcpu, VCPU_REGS_RIP);
  2675. vcpu->arch.regs_dirty = ~0;
  2676. }
  2677. int emulate_instruction(struct kvm_vcpu *vcpu,
  2678. unsigned long cr2,
  2679. u16 error_code,
  2680. int emulation_type)
  2681. {
  2682. int r, shadow_mask;
  2683. struct decode_cache *c;
  2684. struct kvm_run *run = vcpu->run;
  2685. kvm_clear_exception_queue(vcpu);
  2686. vcpu->arch.mmio_fault_cr2 = cr2;
  2687. /*
  2688. * TODO: fix emulate.c to use guest_read/write_register
  2689. * instead of direct ->regs accesses, can save hundred cycles
  2690. * on Intel for instructions that don't read/change RSP, for
  2691. * for example.
  2692. */
  2693. cache_all_regs(vcpu);
  2694. vcpu->mmio_is_write = 0;
  2695. vcpu->arch.pio.string = 0;
  2696. if (!(emulation_type & EMULTYPE_NO_DECODE)) {
  2697. int cs_db, cs_l;
  2698. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  2699. vcpu->arch.emulate_ctxt.vcpu = vcpu;
  2700. vcpu->arch.emulate_ctxt.eflags = kvm_get_rflags(vcpu);
  2701. vcpu->arch.emulate_ctxt.mode =
  2702. (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM)
  2703. ? X86EMUL_MODE_REAL : cs_l
  2704. ? X86EMUL_MODE_PROT64 : cs_db
  2705. ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
  2706. r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
  2707. /* Only allow emulation of specific instructions on #UD
  2708. * (namely VMMCALL, sysenter, sysexit, syscall)*/
  2709. c = &vcpu->arch.emulate_ctxt.decode;
  2710. if (emulation_type & EMULTYPE_TRAP_UD) {
  2711. if (!c->twobyte)
  2712. return EMULATE_FAIL;
  2713. switch (c->b) {
  2714. case 0x01: /* VMMCALL */
  2715. if (c->modrm_mod != 3 || c->modrm_rm != 1)
  2716. return EMULATE_FAIL;
  2717. break;
  2718. case 0x34: /* sysenter */
  2719. case 0x35: /* sysexit */
  2720. if (c->modrm_mod != 0 || c->modrm_rm != 0)
  2721. return EMULATE_FAIL;
  2722. break;
  2723. case 0x05: /* syscall */
  2724. if (c->modrm_mod != 0 || c->modrm_rm != 0)
  2725. return EMULATE_FAIL;
  2726. break;
  2727. default:
  2728. return EMULATE_FAIL;
  2729. }
  2730. if (!(c->modrm_reg == 0 || c->modrm_reg == 3))
  2731. return EMULATE_FAIL;
  2732. }
  2733. ++vcpu->stat.insn_emulation;
  2734. if (r) {
  2735. ++vcpu->stat.insn_emulation_fail;
  2736. if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
  2737. return EMULATE_DONE;
  2738. return EMULATE_FAIL;
  2739. }
  2740. }
  2741. if (emulation_type & EMULTYPE_SKIP) {
  2742. kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.decode.eip);
  2743. return EMULATE_DONE;
  2744. }
  2745. r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
  2746. shadow_mask = vcpu->arch.emulate_ctxt.interruptibility;
  2747. if (r == 0)
  2748. kvm_x86_ops->set_interrupt_shadow(vcpu, shadow_mask);
  2749. if (vcpu->arch.pio.string)
  2750. return EMULATE_DO_MMIO;
  2751. if ((r || vcpu->mmio_is_write) && run) {
  2752. run->exit_reason = KVM_EXIT_MMIO;
  2753. run->mmio.phys_addr = vcpu->mmio_phys_addr;
  2754. memcpy(run->mmio.data, vcpu->mmio_data, 8);
  2755. run->mmio.len = vcpu->mmio_size;
  2756. run->mmio.is_write = vcpu->mmio_is_write;
  2757. }
  2758. if (r) {
  2759. if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
  2760. return EMULATE_DONE;
  2761. if (!vcpu->mmio_needed) {
  2762. kvm_report_emulation_failure(vcpu, "mmio");
  2763. return EMULATE_FAIL;
  2764. }
  2765. return EMULATE_DO_MMIO;
  2766. }
  2767. kvm_set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
  2768. if (vcpu->mmio_is_write) {
  2769. vcpu->mmio_needed = 0;
  2770. return EMULATE_DO_MMIO;
  2771. }
  2772. return EMULATE_DONE;
  2773. }
  2774. EXPORT_SYMBOL_GPL(emulate_instruction);
  2775. static int pio_copy_data(struct kvm_vcpu *vcpu)
  2776. {
  2777. void *p = vcpu->arch.pio_data;
  2778. gva_t q = vcpu->arch.pio.guest_gva;
  2779. unsigned bytes;
  2780. int ret;
  2781. bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count;
  2782. if (vcpu->arch.pio.in)
  2783. ret = kvm_write_guest_virt(q, p, bytes, vcpu);
  2784. else
  2785. ret = kvm_read_guest_virt(q, p, bytes, vcpu);
  2786. return ret;
  2787. }
  2788. int complete_pio(struct kvm_vcpu *vcpu)
  2789. {
  2790. struct kvm_pio_request *io = &vcpu->arch.pio;
  2791. long delta;
  2792. int r;
  2793. unsigned long val;
  2794. if (!io->string) {
  2795. if (io->in) {
  2796. val = kvm_register_read(vcpu, VCPU_REGS_RAX);
  2797. memcpy(&val, vcpu->arch.pio_data, io->size);
  2798. kvm_register_write(vcpu, VCPU_REGS_RAX, val);
  2799. }
  2800. } else {
  2801. if (io->in) {
  2802. r = pio_copy_data(vcpu);
  2803. if (r)
  2804. return r;
  2805. }
  2806. delta = 1;
  2807. if (io->rep) {
  2808. delta *= io->cur_count;
  2809. /*
  2810. * The size of the register should really depend on
  2811. * current address size.
  2812. */
  2813. val = kvm_register_read(vcpu, VCPU_REGS_RCX);
  2814. val -= delta;
  2815. kvm_register_write(vcpu, VCPU_REGS_RCX, val);
  2816. }
  2817. if (io->down)
  2818. delta = -delta;
  2819. delta *= io->size;
  2820. if (io->in) {
  2821. val = kvm_register_read(vcpu, VCPU_REGS_RDI);
  2822. val += delta;
  2823. kvm_register_write(vcpu, VCPU_REGS_RDI, val);
  2824. } else {
  2825. val = kvm_register_read(vcpu, VCPU_REGS_RSI);
  2826. val += delta;
  2827. kvm_register_write(vcpu, VCPU_REGS_RSI, val);
  2828. }
  2829. }
  2830. io->count -= io->cur_count;
  2831. io->cur_count = 0;
  2832. return 0;
  2833. }
  2834. static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
  2835. {
  2836. /* TODO: String I/O for in kernel device */
  2837. int r;
  2838. if (vcpu->arch.pio.in)
  2839. r = kvm_io_bus_read(&vcpu->kvm->pio_bus, vcpu->arch.pio.port,
  2840. vcpu->arch.pio.size, pd);
  2841. else
  2842. r = kvm_io_bus_write(&vcpu->kvm->pio_bus, vcpu->arch.pio.port,
  2843. vcpu->arch.pio.size, pd);
  2844. return r;
  2845. }
  2846. static int pio_string_write(struct kvm_vcpu *vcpu)
  2847. {
  2848. struct kvm_pio_request *io = &vcpu->arch.pio;
  2849. void *pd = vcpu->arch.pio_data;
  2850. int i, r = 0;
  2851. for (i = 0; i < io->cur_count; i++) {
  2852. if (kvm_io_bus_write(&vcpu->kvm->pio_bus,
  2853. io->port, io->size, pd)) {
  2854. r = -EOPNOTSUPP;
  2855. break;
  2856. }
  2857. pd += io->size;
  2858. }
  2859. return r;
  2860. }
  2861. int kvm_emulate_pio(struct kvm_vcpu *vcpu, int in, int size, unsigned port)
  2862. {
  2863. unsigned long val;
  2864. vcpu->run->exit_reason = KVM_EXIT_IO;
  2865. vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  2866. vcpu->run->io.size = vcpu->arch.pio.size = size;
  2867. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  2868. vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1;
  2869. vcpu->run->io.port = vcpu->arch.pio.port = port;
  2870. vcpu->arch.pio.in = in;
  2871. vcpu->arch.pio.string = 0;
  2872. vcpu->arch.pio.down = 0;
  2873. vcpu->arch.pio.rep = 0;
  2874. trace_kvm_pio(vcpu->run->io.direction == KVM_EXIT_IO_OUT, port,
  2875. size, 1);
  2876. val = kvm_register_read(vcpu, VCPU_REGS_RAX);
  2877. memcpy(vcpu->arch.pio_data, &val, 4);
  2878. if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
  2879. complete_pio(vcpu);
  2880. return 1;
  2881. }
  2882. return 0;
  2883. }
  2884. EXPORT_SYMBOL_GPL(kvm_emulate_pio);
  2885. int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, int in,
  2886. int size, unsigned long count, int down,
  2887. gva_t address, int rep, unsigned port)
  2888. {
  2889. unsigned now, in_page;
  2890. int ret = 0;
  2891. vcpu->run->exit_reason = KVM_EXIT_IO;
  2892. vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  2893. vcpu->run->io.size = vcpu->arch.pio.size = size;
  2894. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  2895. vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count;
  2896. vcpu->run->io.port = vcpu->arch.pio.port = port;
  2897. vcpu->arch.pio.in = in;
  2898. vcpu->arch.pio.string = 1;
  2899. vcpu->arch.pio.down = down;
  2900. vcpu->arch.pio.rep = rep;
  2901. trace_kvm_pio(vcpu->run->io.direction == KVM_EXIT_IO_OUT, port,
  2902. size, count);
  2903. if (!count) {
  2904. kvm_x86_ops->skip_emulated_instruction(vcpu);
  2905. return 1;
  2906. }
  2907. if (!down)
  2908. in_page = PAGE_SIZE - offset_in_page(address);
  2909. else
  2910. in_page = offset_in_page(address) + size;
  2911. now = min(count, (unsigned long)in_page / size);
  2912. if (!now)
  2913. now = 1;
  2914. if (down) {
  2915. /*
  2916. * String I/O in reverse. Yuck. Kill the guest, fix later.
  2917. */
  2918. pr_unimpl(vcpu, "guest string pio down\n");
  2919. kvm_inject_gp(vcpu, 0);
  2920. return 1;
  2921. }
  2922. vcpu->run->io.count = now;
  2923. vcpu->arch.pio.cur_count = now;
  2924. if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count)
  2925. kvm_x86_ops->skip_emulated_instruction(vcpu);
  2926. vcpu->arch.pio.guest_gva = address;
  2927. if (!vcpu->arch.pio.in) {
  2928. /* string PIO write */
  2929. ret = pio_copy_data(vcpu);
  2930. if (ret == X86EMUL_PROPAGATE_FAULT) {
  2931. kvm_inject_gp(vcpu, 0);
  2932. return 1;
  2933. }
  2934. if (ret == 0 && !pio_string_write(vcpu)) {
  2935. complete_pio(vcpu);
  2936. if (vcpu->arch.pio.count == 0)
  2937. ret = 1;
  2938. }
  2939. }
  2940. /* no string PIO read support yet */
  2941. return ret;
  2942. }
  2943. EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
  2944. static void bounce_off(void *info)
  2945. {
  2946. /* nothing */
  2947. }
  2948. static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
  2949. void *data)
  2950. {
  2951. struct cpufreq_freqs *freq = data;
  2952. struct kvm *kvm;
  2953. struct kvm_vcpu *vcpu;
  2954. int i, send_ipi = 0;
  2955. if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
  2956. return 0;
  2957. if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
  2958. return 0;
  2959. per_cpu(cpu_tsc_khz, freq->cpu) = freq->new;
  2960. spin_lock(&kvm_lock);
  2961. list_for_each_entry(kvm, &vm_list, vm_list) {
  2962. kvm_for_each_vcpu(i, vcpu, kvm) {
  2963. if (vcpu->cpu != freq->cpu)
  2964. continue;
  2965. if (!kvm_request_guest_time_update(vcpu))
  2966. continue;
  2967. if (vcpu->cpu != smp_processor_id())
  2968. send_ipi++;
  2969. }
  2970. }
  2971. spin_unlock(&kvm_lock);
  2972. if (freq->old < freq->new && send_ipi) {
  2973. /*
  2974. * We upscale the frequency. Must make the guest
  2975. * doesn't see old kvmclock values while running with
  2976. * the new frequency, otherwise we risk the guest sees
  2977. * time go backwards.
  2978. *
  2979. * In case we update the frequency for another cpu
  2980. * (which might be in guest context) send an interrupt
  2981. * to kick the cpu out of guest context. Next time
  2982. * guest context is entered kvmclock will be updated,
  2983. * so the guest will not see stale values.
  2984. */
  2985. smp_call_function_single(freq->cpu, bounce_off, NULL, 1);
  2986. }
  2987. return 0;
  2988. }
  2989. static struct notifier_block kvmclock_cpufreq_notifier_block = {
  2990. .notifier_call = kvmclock_cpufreq_notifier
  2991. };
  2992. static void kvm_timer_init(void)
  2993. {
  2994. int cpu;
  2995. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
  2996. cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
  2997. CPUFREQ_TRANSITION_NOTIFIER);
  2998. for_each_online_cpu(cpu) {
  2999. unsigned long khz = cpufreq_get(cpu);
  3000. if (!khz)
  3001. khz = tsc_khz;
  3002. per_cpu(cpu_tsc_khz, cpu) = khz;
  3003. }
  3004. } else {
  3005. for_each_possible_cpu(cpu)
  3006. per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
  3007. }
  3008. }
  3009. int kvm_arch_init(void *opaque)
  3010. {
  3011. int r;
  3012. struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;
  3013. if (kvm_x86_ops) {
  3014. printk(KERN_ERR "kvm: already loaded the other module\n");
  3015. r = -EEXIST;
  3016. goto out;
  3017. }
  3018. if (!ops->cpu_has_kvm_support()) {
  3019. printk(KERN_ERR "kvm: no hardware support\n");
  3020. r = -EOPNOTSUPP;
  3021. goto out;
  3022. }
  3023. if (ops->disabled_by_bios()) {
  3024. printk(KERN_ERR "kvm: disabled by bios\n");
  3025. r = -EOPNOTSUPP;
  3026. goto out;
  3027. }
  3028. r = kvm_mmu_module_init();
  3029. if (r)
  3030. goto out;
  3031. kvm_init_msr_list();
  3032. kvm_x86_ops = ops;
  3033. kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
  3034. kvm_mmu_set_base_ptes(PT_PRESENT_MASK);
  3035. kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
  3036. PT_DIRTY_MASK, PT64_NX_MASK, 0);
  3037. kvm_timer_init();
  3038. return 0;
  3039. out:
  3040. return r;
  3041. }
  3042. void kvm_arch_exit(void)
  3043. {
  3044. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
  3045. cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
  3046. CPUFREQ_TRANSITION_NOTIFIER);
  3047. kvm_x86_ops = NULL;
  3048. kvm_mmu_module_exit();
  3049. }
  3050. int kvm_emulate_halt(struct kvm_vcpu *vcpu)
  3051. {
  3052. ++vcpu->stat.halt_exits;
  3053. if (irqchip_in_kernel(vcpu->kvm)) {
  3054. vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
  3055. return 1;
  3056. } else {
  3057. vcpu->run->exit_reason = KVM_EXIT_HLT;
  3058. return 0;
  3059. }
  3060. }
  3061. EXPORT_SYMBOL_GPL(kvm_emulate_halt);
  3062. static inline gpa_t hc_gpa(struct kvm_vcpu *vcpu, unsigned long a0,
  3063. unsigned long a1)
  3064. {
  3065. if (is_long_mode(vcpu))
  3066. return a0;
  3067. else
  3068. return a0 | ((gpa_t)a1 << 32);
  3069. }
  3070. int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
  3071. {
  3072. unsigned long nr, a0, a1, a2, a3, ret;
  3073. int r = 1;
  3074. nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3075. a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
  3076. a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3077. a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
  3078. a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
  3079. trace_kvm_hypercall(nr, a0, a1, a2, a3);
  3080. if (!is_long_mode(vcpu)) {
  3081. nr &= 0xFFFFFFFF;
  3082. a0 &= 0xFFFFFFFF;
  3083. a1 &= 0xFFFFFFFF;
  3084. a2 &= 0xFFFFFFFF;
  3085. a3 &= 0xFFFFFFFF;
  3086. }
  3087. if (kvm_x86_ops->get_cpl(vcpu) != 0) {
  3088. ret = -KVM_EPERM;
  3089. goto out;
  3090. }
  3091. switch (nr) {
  3092. case KVM_HC_VAPIC_POLL_IRQ:
  3093. ret = 0;
  3094. break;
  3095. case KVM_HC_MMU_OP:
  3096. r = kvm_pv_mmu_op(vcpu, a0, hc_gpa(vcpu, a1, a2), &ret);
  3097. break;
  3098. default:
  3099. ret = -KVM_ENOSYS;
  3100. break;
  3101. }
  3102. out:
  3103. kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
  3104. ++vcpu->stat.hypercalls;
  3105. return r;
  3106. }
  3107. EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
  3108. int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
  3109. {
  3110. char instruction[3];
  3111. int ret = 0;
  3112. unsigned long rip = kvm_rip_read(vcpu);
  3113. /*
  3114. * Blow out the MMU to ensure that no other VCPU has an active mapping
  3115. * to ensure that the updated hypercall appears atomically across all
  3116. * VCPUs.
  3117. */
  3118. kvm_mmu_zap_all(vcpu->kvm);
  3119. kvm_x86_ops->patch_hypercall(vcpu, instruction);
  3120. if (emulator_write_emulated(rip, instruction, 3, vcpu)
  3121. != X86EMUL_CONTINUE)
  3122. ret = -EFAULT;
  3123. return ret;
  3124. }
  3125. static u64 mk_cr_64(u64 curr_cr, u32 new_val)
  3126. {
  3127. return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
  3128. }
  3129. void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  3130. {
  3131. struct descriptor_table dt = { limit, base };
  3132. kvm_x86_ops->set_gdt(vcpu, &dt);
  3133. }
  3134. void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  3135. {
  3136. struct descriptor_table dt = { limit, base };
  3137. kvm_x86_ops->set_idt(vcpu, &dt);
  3138. }
  3139. void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
  3140. unsigned long *rflags)
  3141. {
  3142. kvm_lmsw(vcpu, msw);
  3143. *rflags = kvm_get_rflags(vcpu);
  3144. }
  3145. unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
  3146. {
  3147. unsigned long value;
  3148. kvm_x86_ops->decache_cr4_guest_bits(vcpu);
  3149. switch (cr) {
  3150. case 0:
  3151. value = vcpu->arch.cr0;
  3152. break;
  3153. case 2:
  3154. value = vcpu->arch.cr2;
  3155. break;
  3156. case 3:
  3157. value = vcpu->arch.cr3;
  3158. break;
  3159. case 4:
  3160. value = vcpu->arch.cr4;
  3161. break;
  3162. case 8:
  3163. value = kvm_get_cr8(vcpu);
  3164. break;
  3165. default:
  3166. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
  3167. return 0;
  3168. }
  3169. return value;
  3170. }
  3171. void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
  3172. unsigned long *rflags)
  3173. {
  3174. switch (cr) {
  3175. case 0:
  3176. kvm_set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val));
  3177. *rflags = kvm_get_rflags(vcpu);
  3178. break;
  3179. case 2:
  3180. vcpu->arch.cr2 = val;
  3181. break;
  3182. case 3:
  3183. kvm_set_cr3(vcpu, val);
  3184. break;
  3185. case 4:
  3186. kvm_set_cr4(vcpu, mk_cr_64(vcpu->arch.cr4, val));
  3187. break;
  3188. case 8:
  3189. kvm_set_cr8(vcpu, val & 0xfUL);
  3190. break;
  3191. default:
  3192. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
  3193. }
  3194. }
  3195. static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
  3196. {
  3197. struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
  3198. int j, nent = vcpu->arch.cpuid_nent;
  3199. e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
  3200. /* when no next entry is found, the current entry[i] is reselected */
  3201. for (j = i + 1; ; j = (j + 1) % nent) {
  3202. struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j];
  3203. if (ej->function == e->function) {
  3204. ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
  3205. return j;
  3206. }
  3207. }
  3208. return 0; /* silence gcc, even though control never reaches here */
  3209. }
  3210. /* find an entry with matching function, matching index (if needed), and that
  3211. * should be read next (if it's stateful) */
  3212. static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
  3213. u32 function, u32 index)
  3214. {
  3215. if (e->function != function)
  3216. return 0;
  3217. if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
  3218. return 0;
  3219. if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
  3220. !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
  3221. return 0;
  3222. return 1;
  3223. }
  3224. struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
  3225. u32 function, u32 index)
  3226. {
  3227. int i;
  3228. struct kvm_cpuid_entry2 *best = NULL;
  3229. for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
  3230. struct kvm_cpuid_entry2 *e;
  3231. e = &vcpu->arch.cpuid_entries[i];
  3232. if (is_matching_cpuid_entry(e, function, index)) {
  3233. if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
  3234. move_to_next_stateful_cpuid_entry(vcpu, i);
  3235. best = e;
  3236. break;
  3237. }
  3238. /*
  3239. * Both basic or both extended?
  3240. */
  3241. if (((e->function ^ function) & 0x80000000) == 0)
  3242. if (!best || e->function > best->function)
  3243. best = e;
  3244. }
  3245. return best;
  3246. }
  3247. int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
  3248. {
  3249. struct kvm_cpuid_entry2 *best;
  3250. best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
  3251. if (best)
  3252. return best->eax & 0xff;
  3253. return 36;
  3254. }
  3255. void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
  3256. {
  3257. u32 function, index;
  3258. struct kvm_cpuid_entry2 *best;
  3259. function = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3260. index = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3261. kvm_register_write(vcpu, VCPU_REGS_RAX, 0);
  3262. kvm_register_write(vcpu, VCPU_REGS_RBX, 0);
  3263. kvm_register_write(vcpu, VCPU_REGS_RCX, 0);
  3264. kvm_register_write(vcpu, VCPU_REGS_RDX, 0);
  3265. best = kvm_find_cpuid_entry(vcpu, function, index);
  3266. if (best) {
  3267. kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax);
  3268. kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx);
  3269. kvm_register_write(vcpu, VCPU_REGS_RCX, best->ecx);
  3270. kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx);
  3271. }
  3272. kvm_x86_ops->skip_emulated_instruction(vcpu);
  3273. trace_kvm_cpuid(function,
  3274. kvm_register_read(vcpu, VCPU_REGS_RAX),
  3275. kvm_register_read(vcpu, VCPU_REGS_RBX),
  3276. kvm_register_read(vcpu, VCPU_REGS_RCX),
  3277. kvm_register_read(vcpu, VCPU_REGS_RDX));
  3278. }
  3279. EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
  3280. /*
  3281. * Check if userspace requested an interrupt window, and that the
  3282. * interrupt window is open.
  3283. *
  3284. * No need to exit to userspace if we already have an interrupt queued.
  3285. */
  3286. static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
  3287. {
  3288. return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
  3289. vcpu->run->request_interrupt_window &&
  3290. kvm_arch_interrupt_allowed(vcpu));
  3291. }
  3292. static void post_kvm_run_save(struct kvm_vcpu *vcpu)
  3293. {
  3294. struct kvm_run *kvm_run = vcpu->run;
  3295. kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
  3296. kvm_run->cr8 = kvm_get_cr8(vcpu);
  3297. kvm_run->apic_base = kvm_get_apic_base(vcpu);
  3298. if (irqchip_in_kernel(vcpu->kvm))
  3299. kvm_run->ready_for_interrupt_injection = 1;
  3300. else
  3301. kvm_run->ready_for_interrupt_injection =
  3302. kvm_arch_interrupt_allowed(vcpu) &&
  3303. !kvm_cpu_has_interrupt(vcpu) &&
  3304. !kvm_event_needs_reinjection(vcpu);
  3305. }
  3306. static void vapic_enter(struct kvm_vcpu *vcpu)
  3307. {
  3308. struct kvm_lapic *apic = vcpu->arch.apic;
  3309. struct page *page;
  3310. if (!apic || !apic->vapic_addr)
  3311. return;
  3312. page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
  3313. vcpu->arch.apic->vapic_page = page;
  3314. }
  3315. static void vapic_exit(struct kvm_vcpu *vcpu)
  3316. {
  3317. struct kvm_lapic *apic = vcpu->arch.apic;
  3318. if (!apic || !apic->vapic_addr)
  3319. return;
  3320. down_read(&vcpu->kvm->slots_lock);
  3321. kvm_release_page_dirty(apic->vapic_page);
  3322. mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
  3323. up_read(&vcpu->kvm->slots_lock);
  3324. }
  3325. static void update_cr8_intercept(struct kvm_vcpu *vcpu)
  3326. {
  3327. int max_irr, tpr;
  3328. if (!kvm_x86_ops->update_cr8_intercept)
  3329. return;
  3330. if (!vcpu->arch.apic)
  3331. return;
  3332. if (!vcpu->arch.apic->vapic_addr)
  3333. max_irr = kvm_lapic_find_highest_irr(vcpu);
  3334. else
  3335. max_irr = -1;
  3336. if (max_irr != -1)
  3337. max_irr >>= 4;
  3338. tpr = kvm_lapic_get_cr8(vcpu);
  3339. kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
  3340. }
  3341. static void inject_pending_event(struct kvm_vcpu *vcpu)
  3342. {
  3343. /* try to reinject previous events if any */
  3344. if (vcpu->arch.exception.pending) {
  3345. kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
  3346. vcpu->arch.exception.has_error_code,
  3347. vcpu->arch.exception.error_code);
  3348. return;
  3349. }
  3350. if (vcpu->arch.nmi_injected) {
  3351. kvm_x86_ops->set_nmi(vcpu);
  3352. return;
  3353. }
  3354. if (vcpu->arch.interrupt.pending) {
  3355. kvm_x86_ops->set_irq(vcpu);
  3356. return;
  3357. }
  3358. /* try to inject new event if pending */
  3359. if (vcpu->arch.nmi_pending) {
  3360. if (kvm_x86_ops->nmi_allowed(vcpu)) {
  3361. vcpu->arch.nmi_pending = false;
  3362. vcpu->arch.nmi_injected = true;
  3363. kvm_x86_ops->set_nmi(vcpu);
  3364. }
  3365. } else if (kvm_cpu_has_interrupt(vcpu)) {
  3366. if (kvm_x86_ops->interrupt_allowed(vcpu)) {
  3367. kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
  3368. false);
  3369. kvm_x86_ops->set_irq(vcpu);
  3370. }
  3371. }
  3372. }
  3373. static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
  3374. {
  3375. int r;
  3376. bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
  3377. vcpu->run->request_interrupt_window;
  3378. if (vcpu->requests)
  3379. if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
  3380. kvm_mmu_unload(vcpu);
  3381. r = kvm_mmu_reload(vcpu);
  3382. if (unlikely(r))
  3383. goto out;
  3384. if (vcpu->requests) {
  3385. if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests))
  3386. __kvm_migrate_timers(vcpu);
  3387. if (test_and_clear_bit(KVM_REQ_KVMCLOCK_UPDATE, &vcpu->requests))
  3388. kvm_write_guest_time(vcpu);
  3389. if (test_and_clear_bit(KVM_REQ_MMU_SYNC, &vcpu->requests))
  3390. kvm_mmu_sync_roots(vcpu);
  3391. if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
  3392. kvm_x86_ops->tlb_flush(vcpu);
  3393. if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS,
  3394. &vcpu->requests)) {
  3395. vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
  3396. r = 0;
  3397. goto out;
  3398. }
  3399. if (test_and_clear_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests)) {
  3400. vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
  3401. r = 0;
  3402. goto out;
  3403. }
  3404. }
  3405. preempt_disable();
  3406. kvm_x86_ops->prepare_guest_switch(vcpu);
  3407. kvm_load_guest_fpu(vcpu);
  3408. local_irq_disable();
  3409. clear_bit(KVM_REQ_KICK, &vcpu->requests);
  3410. smp_mb__after_clear_bit();
  3411. if (vcpu->requests || need_resched() || signal_pending(current)) {
  3412. set_bit(KVM_REQ_KICK, &vcpu->requests);
  3413. local_irq_enable();
  3414. preempt_enable();
  3415. r = 1;
  3416. goto out;
  3417. }
  3418. inject_pending_event(vcpu);
  3419. /* enable NMI/IRQ window open exits if needed */
  3420. if (vcpu->arch.nmi_pending)
  3421. kvm_x86_ops->enable_nmi_window(vcpu);
  3422. else if (kvm_cpu_has_interrupt(vcpu) || req_int_win)
  3423. kvm_x86_ops->enable_irq_window(vcpu);
  3424. if (kvm_lapic_enabled(vcpu)) {
  3425. update_cr8_intercept(vcpu);
  3426. kvm_lapic_sync_to_vapic(vcpu);
  3427. }
  3428. up_read(&vcpu->kvm->slots_lock);
  3429. kvm_guest_enter();
  3430. if (unlikely(vcpu->arch.switch_db_regs)) {
  3431. set_debugreg(0, 7);
  3432. set_debugreg(vcpu->arch.eff_db[0], 0);
  3433. set_debugreg(vcpu->arch.eff_db[1], 1);
  3434. set_debugreg(vcpu->arch.eff_db[2], 2);
  3435. set_debugreg(vcpu->arch.eff_db[3], 3);
  3436. }
  3437. trace_kvm_entry(vcpu->vcpu_id);
  3438. kvm_x86_ops->run(vcpu);
  3439. /*
  3440. * If the guest has used debug registers, at least dr7
  3441. * will be disabled while returning to the host.
  3442. * If we don't have active breakpoints in the host, we don't
  3443. * care about the messed up debug address registers. But if
  3444. * we have some of them active, restore the old state.
  3445. */
  3446. if (hw_breakpoint_active())
  3447. hw_breakpoint_restore();
  3448. set_bit(KVM_REQ_KICK, &vcpu->requests);
  3449. local_irq_enable();
  3450. ++vcpu->stat.exits;
  3451. /*
  3452. * We must have an instruction between local_irq_enable() and
  3453. * kvm_guest_exit(), so the timer interrupt isn't delayed by
  3454. * the interrupt shadow. The stat.exits increment will do nicely.
  3455. * But we need to prevent reordering, hence this barrier():
  3456. */
  3457. barrier();
  3458. kvm_guest_exit();
  3459. preempt_enable();
  3460. down_read(&vcpu->kvm->slots_lock);
  3461. /*
  3462. * Profile KVM exit RIPs:
  3463. */
  3464. if (unlikely(prof_on == KVM_PROFILING)) {
  3465. unsigned long rip = kvm_rip_read(vcpu);
  3466. profile_hit(KVM_PROFILING, (void *)rip);
  3467. }
  3468. kvm_lapic_sync_from_vapic(vcpu);
  3469. r = kvm_x86_ops->handle_exit(vcpu);
  3470. out:
  3471. return r;
  3472. }
  3473. static int __vcpu_run(struct kvm_vcpu *vcpu)
  3474. {
  3475. int r;
  3476. if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) {
  3477. pr_debug("vcpu %d received sipi with vector # %x\n",
  3478. vcpu->vcpu_id, vcpu->arch.sipi_vector);
  3479. kvm_lapic_reset(vcpu);
  3480. r = kvm_arch_vcpu_reset(vcpu);
  3481. if (r)
  3482. return r;
  3483. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  3484. }
  3485. down_read(&vcpu->kvm->slots_lock);
  3486. vapic_enter(vcpu);
  3487. r = 1;
  3488. while (r > 0) {
  3489. if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE)
  3490. r = vcpu_enter_guest(vcpu);
  3491. else {
  3492. up_read(&vcpu->kvm->slots_lock);
  3493. kvm_vcpu_block(vcpu);
  3494. down_read(&vcpu->kvm->slots_lock);
  3495. if (test_and_clear_bit(KVM_REQ_UNHALT, &vcpu->requests))
  3496. {
  3497. switch(vcpu->arch.mp_state) {
  3498. case KVM_MP_STATE_HALTED:
  3499. vcpu->arch.mp_state =
  3500. KVM_MP_STATE_RUNNABLE;
  3501. case KVM_MP_STATE_RUNNABLE:
  3502. break;
  3503. case KVM_MP_STATE_SIPI_RECEIVED:
  3504. default:
  3505. r = -EINTR;
  3506. break;
  3507. }
  3508. }
  3509. }
  3510. if (r <= 0)
  3511. break;
  3512. clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
  3513. if (kvm_cpu_has_pending_timer(vcpu))
  3514. kvm_inject_pending_timer_irqs(vcpu);
  3515. if (dm_request_for_irq_injection(vcpu)) {
  3516. r = -EINTR;
  3517. vcpu->run->exit_reason = KVM_EXIT_INTR;
  3518. ++vcpu->stat.request_irq_exits;
  3519. }
  3520. if (signal_pending(current)) {
  3521. r = -EINTR;
  3522. vcpu->run->exit_reason = KVM_EXIT_INTR;
  3523. ++vcpu->stat.signal_exits;
  3524. }
  3525. if (need_resched()) {
  3526. up_read(&vcpu->kvm->slots_lock);
  3527. kvm_resched(vcpu);
  3528. down_read(&vcpu->kvm->slots_lock);
  3529. }
  3530. }
  3531. up_read(&vcpu->kvm->slots_lock);
  3532. post_kvm_run_save(vcpu);
  3533. vapic_exit(vcpu);
  3534. return r;
  3535. }
  3536. int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  3537. {
  3538. int r;
  3539. sigset_t sigsaved;
  3540. vcpu_load(vcpu);
  3541. if (vcpu->sigset_active)
  3542. sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
  3543. if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
  3544. kvm_vcpu_block(vcpu);
  3545. clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
  3546. r = -EAGAIN;
  3547. goto out;
  3548. }
  3549. /* re-sync apic's tpr */
  3550. if (!irqchip_in_kernel(vcpu->kvm))
  3551. kvm_set_cr8(vcpu, kvm_run->cr8);
  3552. if (vcpu->arch.pio.cur_count) {
  3553. r = complete_pio(vcpu);
  3554. if (r)
  3555. goto out;
  3556. }
  3557. if (vcpu->mmio_needed) {
  3558. memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
  3559. vcpu->mmio_read_completed = 1;
  3560. vcpu->mmio_needed = 0;
  3561. down_read(&vcpu->kvm->slots_lock);
  3562. r = emulate_instruction(vcpu, vcpu->arch.mmio_fault_cr2, 0,
  3563. EMULTYPE_NO_DECODE);
  3564. up_read(&vcpu->kvm->slots_lock);
  3565. if (r == EMULATE_DO_MMIO) {
  3566. /*
  3567. * Read-modify-write. Back to userspace.
  3568. */
  3569. r = 0;
  3570. goto out;
  3571. }
  3572. }
  3573. if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL)
  3574. kvm_register_write(vcpu, VCPU_REGS_RAX,
  3575. kvm_run->hypercall.ret);
  3576. r = __vcpu_run(vcpu);
  3577. out:
  3578. if (vcpu->sigset_active)
  3579. sigprocmask(SIG_SETMASK, &sigsaved, NULL);
  3580. vcpu_put(vcpu);
  3581. return r;
  3582. }
  3583. int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  3584. {
  3585. vcpu_load(vcpu);
  3586. regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3587. regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  3588. regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3589. regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  3590. regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
  3591. regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
  3592. regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  3593. regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  3594. #ifdef CONFIG_X86_64
  3595. regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
  3596. regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
  3597. regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
  3598. regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
  3599. regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
  3600. regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
  3601. regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
  3602. regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
  3603. #endif
  3604. regs->rip = kvm_rip_read(vcpu);
  3605. regs->rflags = kvm_get_rflags(vcpu);
  3606. vcpu_put(vcpu);
  3607. return 0;
  3608. }
  3609. int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  3610. {
  3611. vcpu_load(vcpu);
  3612. kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
  3613. kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
  3614. kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
  3615. kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
  3616. kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
  3617. kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
  3618. kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
  3619. kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
  3620. #ifdef CONFIG_X86_64
  3621. kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
  3622. kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
  3623. kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
  3624. kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
  3625. kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
  3626. kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
  3627. kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
  3628. kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
  3629. #endif
  3630. kvm_rip_write(vcpu, regs->rip);
  3631. kvm_set_rflags(vcpu, regs->rflags);
  3632. vcpu->arch.exception.pending = false;
  3633. vcpu_put(vcpu);
  3634. return 0;
  3635. }
  3636. void kvm_get_segment(struct kvm_vcpu *vcpu,
  3637. struct kvm_segment *var, int seg)
  3638. {
  3639. kvm_x86_ops->get_segment(vcpu, var, seg);
  3640. }
  3641. void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
  3642. {
  3643. struct kvm_segment cs;
  3644. kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
  3645. *db = cs.db;
  3646. *l = cs.l;
  3647. }
  3648. EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
  3649. int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  3650. struct kvm_sregs *sregs)
  3651. {
  3652. struct descriptor_table dt;
  3653. vcpu_load(vcpu);
  3654. kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  3655. kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  3656. kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  3657. kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  3658. kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  3659. kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  3660. kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  3661. kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  3662. kvm_x86_ops->get_idt(vcpu, &dt);
  3663. sregs->idt.limit = dt.limit;
  3664. sregs->idt.base = dt.base;
  3665. kvm_x86_ops->get_gdt(vcpu, &dt);
  3666. sregs->gdt.limit = dt.limit;
  3667. sregs->gdt.base = dt.base;
  3668. kvm_x86_ops->decache_cr4_guest_bits(vcpu);
  3669. sregs->cr0 = vcpu->arch.cr0;
  3670. sregs->cr2 = vcpu->arch.cr2;
  3671. sregs->cr3 = vcpu->arch.cr3;
  3672. sregs->cr4 = vcpu->arch.cr4;
  3673. sregs->cr8 = kvm_get_cr8(vcpu);
  3674. sregs->efer = vcpu->arch.shadow_efer;
  3675. sregs->apic_base = kvm_get_apic_base(vcpu);
  3676. memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
  3677. if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
  3678. set_bit(vcpu->arch.interrupt.nr,
  3679. (unsigned long *)sregs->interrupt_bitmap);
  3680. vcpu_put(vcpu);
  3681. return 0;
  3682. }
  3683. int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
  3684. struct kvm_mp_state *mp_state)
  3685. {
  3686. vcpu_load(vcpu);
  3687. mp_state->mp_state = vcpu->arch.mp_state;
  3688. vcpu_put(vcpu);
  3689. return 0;
  3690. }
  3691. int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
  3692. struct kvm_mp_state *mp_state)
  3693. {
  3694. vcpu_load(vcpu);
  3695. vcpu->arch.mp_state = mp_state->mp_state;
  3696. vcpu_put(vcpu);
  3697. return 0;
  3698. }
  3699. static void kvm_set_segment(struct kvm_vcpu *vcpu,
  3700. struct kvm_segment *var, int seg)
  3701. {
  3702. kvm_x86_ops->set_segment(vcpu, var, seg);
  3703. }
  3704. static void seg_desct_to_kvm_desct(struct desc_struct *seg_desc, u16 selector,
  3705. struct kvm_segment *kvm_desct)
  3706. {
  3707. kvm_desct->base = get_desc_base(seg_desc);
  3708. kvm_desct->limit = get_desc_limit(seg_desc);
  3709. if (seg_desc->g) {
  3710. kvm_desct->limit <<= 12;
  3711. kvm_desct->limit |= 0xfff;
  3712. }
  3713. kvm_desct->selector = selector;
  3714. kvm_desct->type = seg_desc->type;
  3715. kvm_desct->present = seg_desc->p;
  3716. kvm_desct->dpl = seg_desc->dpl;
  3717. kvm_desct->db = seg_desc->d;
  3718. kvm_desct->s = seg_desc->s;
  3719. kvm_desct->l = seg_desc->l;
  3720. kvm_desct->g = seg_desc->g;
  3721. kvm_desct->avl = seg_desc->avl;
  3722. if (!selector)
  3723. kvm_desct->unusable = 1;
  3724. else
  3725. kvm_desct->unusable = 0;
  3726. kvm_desct->padding = 0;
  3727. }
  3728. static void get_segment_descriptor_dtable(struct kvm_vcpu *vcpu,
  3729. u16 selector,
  3730. struct descriptor_table *dtable)
  3731. {
  3732. if (selector & 1 << 2) {
  3733. struct kvm_segment kvm_seg;
  3734. kvm_get_segment(vcpu, &kvm_seg, VCPU_SREG_LDTR);
  3735. if (kvm_seg.unusable)
  3736. dtable->limit = 0;
  3737. else
  3738. dtable->limit = kvm_seg.limit;
  3739. dtable->base = kvm_seg.base;
  3740. }
  3741. else
  3742. kvm_x86_ops->get_gdt(vcpu, dtable);
  3743. }
  3744. /* allowed just for 8 bytes segments */
  3745. static int load_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
  3746. struct desc_struct *seg_desc)
  3747. {
  3748. struct descriptor_table dtable;
  3749. u16 index = selector >> 3;
  3750. get_segment_descriptor_dtable(vcpu, selector, &dtable);
  3751. if (dtable.limit < index * 8 + 7) {
  3752. kvm_queue_exception_e(vcpu, GP_VECTOR, selector & 0xfffc);
  3753. return 1;
  3754. }
  3755. return kvm_read_guest_virt(dtable.base + index*8, seg_desc, sizeof(*seg_desc), vcpu);
  3756. }
  3757. /* allowed just for 8 bytes segments */
  3758. static int save_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
  3759. struct desc_struct *seg_desc)
  3760. {
  3761. struct descriptor_table dtable;
  3762. u16 index = selector >> 3;
  3763. get_segment_descriptor_dtable(vcpu, selector, &dtable);
  3764. if (dtable.limit < index * 8 + 7)
  3765. return 1;
  3766. return kvm_write_guest_virt(dtable.base + index*8, seg_desc, sizeof(*seg_desc), vcpu);
  3767. }
  3768. static gpa_t get_tss_base_addr(struct kvm_vcpu *vcpu,
  3769. struct desc_struct *seg_desc)
  3770. {
  3771. u32 base_addr = get_desc_base(seg_desc);
  3772. return vcpu->arch.mmu.gva_to_gpa(vcpu, base_addr);
  3773. }
  3774. static u16 get_segment_selector(struct kvm_vcpu *vcpu, int seg)
  3775. {
  3776. struct kvm_segment kvm_seg;
  3777. kvm_get_segment(vcpu, &kvm_seg, seg);
  3778. return kvm_seg.selector;
  3779. }
  3780. static int load_segment_descriptor_to_kvm_desct(struct kvm_vcpu *vcpu,
  3781. u16 selector,
  3782. struct kvm_segment *kvm_seg)
  3783. {
  3784. struct desc_struct seg_desc;
  3785. if (load_guest_segment_descriptor(vcpu, selector, &seg_desc))
  3786. return 1;
  3787. seg_desct_to_kvm_desct(&seg_desc, selector, kvm_seg);
  3788. return 0;
  3789. }
  3790. static int kvm_load_realmode_segment(struct kvm_vcpu *vcpu, u16 selector, int seg)
  3791. {
  3792. struct kvm_segment segvar = {
  3793. .base = selector << 4,
  3794. .limit = 0xffff,
  3795. .selector = selector,
  3796. .type = 3,
  3797. .present = 1,
  3798. .dpl = 3,
  3799. .db = 0,
  3800. .s = 1,
  3801. .l = 0,
  3802. .g = 0,
  3803. .avl = 0,
  3804. .unusable = 0,
  3805. };
  3806. kvm_x86_ops->set_segment(vcpu, &segvar, seg);
  3807. return 0;
  3808. }
  3809. static int is_vm86_segment(struct kvm_vcpu *vcpu, int seg)
  3810. {
  3811. return (seg != VCPU_SREG_LDTR) &&
  3812. (seg != VCPU_SREG_TR) &&
  3813. (kvm_get_rflags(vcpu) & X86_EFLAGS_VM);
  3814. }
  3815. int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
  3816. int type_bits, int seg)
  3817. {
  3818. struct kvm_segment kvm_seg;
  3819. if (is_vm86_segment(vcpu, seg) || !(vcpu->arch.cr0 & X86_CR0_PE))
  3820. return kvm_load_realmode_segment(vcpu, selector, seg);
  3821. if (load_segment_descriptor_to_kvm_desct(vcpu, selector, &kvm_seg))
  3822. return 1;
  3823. kvm_seg.type |= type_bits;
  3824. if (seg != VCPU_SREG_SS && seg != VCPU_SREG_CS &&
  3825. seg != VCPU_SREG_LDTR)
  3826. if (!kvm_seg.s)
  3827. kvm_seg.unusable = 1;
  3828. kvm_set_segment(vcpu, &kvm_seg, seg);
  3829. return 0;
  3830. }
  3831. static void save_state_to_tss32(struct kvm_vcpu *vcpu,
  3832. struct tss_segment_32 *tss)
  3833. {
  3834. tss->cr3 = vcpu->arch.cr3;
  3835. tss->eip = kvm_rip_read(vcpu);
  3836. tss->eflags = kvm_get_rflags(vcpu);
  3837. tss->eax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3838. tss->ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3839. tss->edx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  3840. tss->ebx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  3841. tss->esp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  3842. tss->ebp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  3843. tss->esi = kvm_register_read(vcpu, VCPU_REGS_RSI);
  3844. tss->edi = kvm_register_read(vcpu, VCPU_REGS_RDI);
  3845. tss->es = get_segment_selector(vcpu, VCPU_SREG_ES);
  3846. tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS);
  3847. tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS);
  3848. tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS);
  3849. tss->fs = get_segment_selector(vcpu, VCPU_SREG_FS);
  3850. tss->gs = get_segment_selector(vcpu, VCPU_SREG_GS);
  3851. tss->ldt_selector = get_segment_selector(vcpu, VCPU_SREG_LDTR);
  3852. }
  3853. static int load_state_from_tss32(struct kvm_vcpu *vcpu,
  3854. struct tss_segment_32 *tss)
  3855. {
  3856. kvm_set_cr3(vcpu, tss->cr3);
  3857. kvm_rip_write(vcpu, tss->eip);
  3858. kvm_set_rflags(vcpu, tss->eflags | 2);
  3859. kvm_register_write(vcpu, VCPU_REGS_RAX, tss->eax);
  3860. kvm_register_write(vcpu, VCPU_REGS_RCX, tss->ecx);
  3861. kvm_register_write(vcpu, VCPU_REGS_RDX, tss->edx);
  3862. kvm_register_write(vcpu, VCPU_REGS_RBX, tss->ebx);
  3863. kvm_register_write(vcpu, VCPU_REGS_RSP, tss->esp);
  3864. kvm_register_write(vcpu, VCPU_REGS_RBP, tss->ebp);
  3865. kvm_register_write(vcpu, VCPU_REGS_RSI, tss->esi);
  3866. kvm_register_write(vcpu, VCPU_REGS_RDI, tss->edi);
  3867. if (kvm_load_segment_descriptor(vcpu, tss->ldt_selector, 0, VCPU_SREG_LDTR))
  3868. return 1;
  3869. if (kvm_load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES))
  3870. return 1;
  3871. if (kvm_load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS))
  3872. return 1;
  3873. if (kvm_load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS))
  3874. return 1;
  3875. if (kvm_load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS))
  3876. return 1;
  3877. if (kvm_load_segment_descriptor(vcpu, tss->fs, 1, VCPU_SREG_FS))
  3878. return 1;
  3879. if (kvm_load_segment_descriptor(vcpu, tss->gs, 1, VCPU_SREG_GS))
  3880. return 1;
  3881. return 0;
  3882. }
  3883. static void save_state_to_tss16(struct kvm_vcpu *vcpu,
  3884. struct tss_segment_16 *tss)
  3885. {
  3886. tss->ip = kvm_rip_read(vcpu);
  3887. tss->flag = kvm_get_rflags(vcpu);
  3888. tss->ax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3889. tss->cx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3890. tss->dx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  3891. tss->bx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  3892. tss->sp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  3893. tss->bp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  3894. tss->si = kvm_register_read(vcpu, VCPU_REGS_RSI);
  3895. tss->di = kvm_register_read(vcpu, VCPU_REGS_RDI);
  3896. tss->es = get_segment_selector(vcpu, VCPU_SREG_ES);
  3897. tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS);
  3898. tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS);
  3899. tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS);
  3900. tss->ldt = get_segment_selector(vcpu, VCPU_SREG_LDTR);
  3901. }
  3902. static int load_state_from_tss16(struct kvm_vcpu *vcpu,
  3903. struct tss_segment_16 *tss)
  3904. {
  3905. kvm_rip_write(vcpu, tss->ip);
  3906. kvm_set_rflags(vcpu, tss->flag | 2);
  3907. kvm_register_write(vcpu, VCPU_REGS_RAX, tss->ax);
  3908. kvm_register_write(vcpu, VCPU_REGS_RCX, tss->cx);
  3909. kvm_register_write(vcpu, VCPU_REGS_RDX, tss->dx);
  3910. kvm_register_write(vcpu, VCPU_REGS_RBX, tss->bx);
  3911. kvm_register_write(vcpu, VCPU_REGS_RSP, tss->sp);
  3912. kvm_register_write(vcpu, VCPU_REGS_RBP, tss->bp);
  3913. kvm_register_write(vcpu, VCPU_REGS_RSI, tss->si);
  3914. kvm_register_write(vcpu, VCPU_REGS_RDI, tss->di);
  3915. if (kvm_load_segment_descriptor(vcpu, tss->ldt, 0, VCPU_SREG_LDTR))
  3916. return 1;
  3917. if (kvm_load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES))
  3918. return 1;
  3919. if (kvm_load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS))
  3920. return 1;
  3921. if (kvm_load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS))
  3922. return 1;
  3923. if (kvm_load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS))
  3924. return 1;
  3925. return 0;
  3926. }
  3927. static int kvm_task_switch_16(struct kvm_vcpu *vcpu, u16 tss_selector,
  3928. u16 old_tss_sel, u32 old_tss_base,
  3929. struct desc_struct *nseg_desc)
  3930. {
  3931. struct tss_segment_16 tss_segment_16;
  3932. int ret = 0;
  3933. if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_16,
  3934. sizeof tss_segment_16))
  3935. goto out;
  3936. save_state_to_tss16(vcpu, &tss_segment_16);
  3937. if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_16,
  3938. sizeof tss_segment_16))
  3939. goto out;
  3940. if (kvm_read_guest(vcpu->kvm, get_tss_base_addr(vcpu, nseg_desc),
  3941. &tss_segment_16, sizeof tss_segment_16))
  3942. goto out;
  3943. if (old_tss_sel != 0xffff) {
  3944. tss_segment_16.prev_task_link = old_tss_sel;
  3945. if (kvm_write_guest(vcpu->kvm,
  3946. get_tss_base_addr(vcpu, nseg_desc),
  3947. &tss_segment_16.prev_task_link,
  3948. sizeof tss_segment_16.prev_task_link))
  3949. goto out;
  3950. }
  3951. if (load_state_from_tss16(vcpu, &tss_segment_16))
  3952. goto out;
  3953. ret = 1;
  3954. out:
  3955. return ret;
  3956. }
  3957. static int kvm_task_switch_32(struct kvm_vcpu *vcpu, u16 tss_selector,
  3958. u16 old_tss_sel, u32 old_tss_base,
  3959. struct desc_struct *nseg_desc)
  3960. {
  3961. struct tss_segment_32 tss_segment_32;
  3962. int ret = 0;
  3963. if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_32,
  3964. sizeof tss_segment_32))
  3965. goto out;
  3966. save_state_to_tss32(vcpu, &tss_segment_32);
  3967. if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_32,
  3968. sizeof tss_segment_32))
  3969. goto out;
  3970. if (kvm_read_guest(vcpu->kvm, get_tss_base_addr(vcpu, nseg_desc),
  3971. &tss_segment_32, sizeof tss_segment_32))
  3972. goto out;
  3973. if (old_tss_sel != 0xffff) {
  3974. tss_segment_32.prev_task_link = old_tss_sel;
  3975. if (kvm_write_guest(vcpu->kvm,
  3976. get_tss_base_addr(vcpu, nseg_desc),
  3977. &tss_segment_32.prev_task_link,
  3978. sizeof tss_segment_32.prev_task_link))
  3979. goto out;
  3980. }
  3981. if (load_state_from_tss32(vcpu, &tss_segment_32))
  3982. goto out;
  3983. ret = 1;
  3984. out:
  3985. return ret;
  3986. }
  3987. int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason)
  3988. {
  3989. struct kvm_segment tr_seg;
  3990. struct desc_struct cseg_desc;
  3991. struct desc_struct nseg_desc;
  3992. int ret = 0;
  3993. u32 old_tss_base = get_segment_base(vcpu, VCPU_SREG_TR);
  3994. u16 old_tss_sel = get_segment_selector(vcpu, VCPU_SREG_TR);
  3995. old_tss_base = vcpu->arch.mmu.gva_to_gpa(vcpu, old_tss_base);
  3996. /* FIXME: Handle errors. Failure to read either TSS or their
  3997. * descriptors should generate a pagefault.
  3998. */
  3999. if (load_guest_segment_descriptor(vcpu, tss_selector, &nseg_desc))
  4000. goto out;
  4001. if (load_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc))
  4002. goto out;
  4003. if (reason != TASK_SWITCH_IRET) {
  4004. int cpl;
  4005. cpl = kvm_x86_ops->get_cpl(vcpu);
  4006. if ((tss_selector & 3) > nseg_desc.dpl || cpl > nseg_desc.dpl) {
  4007. kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
  4008. return 1;
  4009. }
  4010. }
  4011. if (!nseg_desc.p || get_desc_limit(&nseg_desc) < 0x67) {
  4012. kvm_queue_exception_e(vcpu, TS_VECTOR, tss_selector & 0xfffc);
  4013. return 1;
  4014. }
  4015. if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
  4016. cseg_desc.type &= ~(1 << 1); //clear the B flag
  4017. save_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc);
  4018. }
  4019. if (reason == TASK_SWITCH_IRET) {
  4020. u32 eflags = kvm_get_rflags(vcpu);
  4021. kvm_set_rflags(vcpu, eflags & ~X86_EFLAGS_NT);
  4022. }
  4023. /* set back link to prev task only if NT bit is set in eflags
  4024. note that old_tss_sel is not used afetr this point */
  4025. if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
  4026. old_tss_sel = 0xffff;
  4027. if (nseg_desc.type & 8)
  4028. ret = kvm_task_switch_32(vcpu, tss_selector, old_tss_sel,
  4029. old_tss_base, &nseg_desc);
  4030. else
  4031. ret = kvm_task_switch_16(vcpu, tss_selector, old_tss_sel,
  4032. old_tss_base, &nseg_desc);
  4033. if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE) {
  4034. u32 eflags = kvm_get_rflags(vcpu);
  4035. kvm_set_rflags(vcpu, eflags | X86_EFLAGS_NT);
  4036. }
  4037. if (reason != TASK_SWITCH_IRET) {
  4038. nseg_desc.type |= (1 << 1);
  4039. save_guest_segment_descriptor(vcpu, tss_selector,
  4040. &nseg_desc);
  4041. }
  4042. kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 | X86_CR0_TS);
  4043. seg_desct_to_kvm_desct(&nseg_desc, tss_selector, &tr_seg);
  4044. tr_seg.type = 11;
  4045. kvm_set_segment(vcpu, &tr_seg, VCPU_SREG_TR);
  4046. out:
  4047. return ret;
  4048. }
  4049. EXPORT_SYMBOL_GPL(kvm_task_switch);
  4050. int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  4051. struct kvm_sregs *sregs)
  4052. {
  4053. int mmu_reset_needed = 0;
  4054. int pending_vec, max_bits;
  4055. struct descriptor_table dt;
  4056. vcpu_load(vcpu);
  4057. dt.limit = sregs->idt.limit;
  4058. dt.base = sregs->idt.base;
  4059. kvm_x86_ops->set_idt(vcpu, &dt);
  4060. dt.limit = sregs->gdt.limit;
  4061. dt.base = sregs->gdt.base;
  4062. kvm_x86_ops->set_gdt(vcpu, &dt);
  4063. vcpu->arch.cr2 = sregs->cr2;
  4064. mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
  4065. vcpu->arch.cr3 = sregs->cr3;
  4066. kvm_set_cr8(vcpu, sregs->cr8);
  4067. mmu_reset_needed |= vcpu->arch.shadow_efer != sregs->efer;
  4068. kvm_x86_ops->set_efer(vcpu, sregs->efer);
  4069. kvm_set_apic_base(vcpu, sregs->apic_base);
  4070. kvm_x86_ops->decache_cr4_guest_bits(vcpu);
  4071. mmu_reset_needed |= vcpu->arch.cr0 != sregs->cr0;
  4072. kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
  4073. vcpu->arch.cr0 = sregs->cr0;
  4074. mmu_reset_needed |= vcpu->arch.cr4 != sregs->cr4;
  4075. kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
  4076. if (!is_long_mode(vcpu) && is_pae(vcpu)) {
  4077. load_pdptrs(vcpu, vcpu->arch.cr3);
  4078. mmu_reset_needed = 1;
  4079. }
  4080. if (mmu_reset_needed)
  4081. kvm_mmu_reset_context(vcpu);
  4082. max_bits = (sizeof sregs->interrupt_bitmap) << 3;
  4083. pending_vec = find_first_bit(
  4084. (const unsigned long *)sregs->interrupt_bitmap, max_bits);
  4085. if (pending_vec < max_bits) {
  4086. kvm_queue_interrupt(vcpu, pending_vec, false);
  4087. pr_debug("Set back pending irq %d\n", pending_vec);
  4088. if (irqchip_in_kernel(vcpu->kvm))
  4089. kvm_pic_clear_isr_ack(vcpu->kvm);
  4090. }
  4091. kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  4092. kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  4093. kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  4094. kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  4095. kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  4096. kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  4097. kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  4098. kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  4099. update_cr8_intercept(vcpu);
  4100. /* Older userspace won't unhalt the vcpu on reset. */
  4101. if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
  4102. sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
  4103. !(vcpu->arch.cr0 & X86_CR0_PE))
  4104. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  4105. vcpu_put(vcpu);
  4106. return 0;
  4107. }
  4108. int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
  4109. struct kvm_guest_debug *dbg)
  4110. {
  4111. unsigned long rflags;
  4112. int i, r;
  4113. vcpu_load(vcpu);
  4114. if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
  4115. r = -EBUSY;
  4116. if (vcpu->arch.exception.pending)
  4117. goto unlock_out;
  4118. if (dbg->control & KVM_GUESTDBG_INJECT_DB)
  4119. kvm_queue_exception(vcpu, DB_VECTOR);
  4120. else
  4121. kvm_queue_exception(vcpu, BP_VECTOR);
  4122. }
  4123. /*
  4124. * Read rflags as long as potentially injected trace flags are still
  4125. * filtered out.
  4126. */
  4127. rflags = kvm_get_rflags(vcpu);
  4128. vcpu->guest_debug = dbg->control;
  4129. if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
  4130. vcpu->guest_debug = 0;
  4131. if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
  4132. for (i = 0; i < KVM_NR_DB_REGS; ++i)
  4133. vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
  4134. vcpu->arch.switch_db_regs =
  4135. (dbg->arch.debugreg[7] & DR7_BP_EN_MASK);
  4136. } else {
  4137. for (i = 0; i < KVM_NR_DB_REGS; i++)
  4138. vcpu->arch.eff_db[i] = vcpu->arch.db[i];
  4139. vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK);
  4140. }
  4141. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
  4142. vcpu->arch.singlestep_cs =
  4143. get_segment_selector(vcpu, VCPU_SREG_CS);
  4144. vcpu->arch.singlestep_rip = kvm_rip_read(vcpu);
  4145. }
  4146. /*
  4147. * Trigger an rflags update that will inject or remove the trace
  4148. * flags.
  4149. */
  4150. kvm_set_rflags(vcpu, rflags);
  4151. kvm_x86_ops->set_guest_debug(vcpu, dbg);
  4152. r = 0;
  4153. unlock_out:
  4154. vcpu_put(vcpu);
  4155. return r;
  4156. }
  4157. /*
  4158. * fxsave fpu state. Taken from x86_64/processor.h. To be killed when
  4159. * we have asm/x86/processor.h
  4160. */
  4161. struct fxsave {
  4162. u16 cwd;
  4163. u16 swd;
  4164. u16 twd;
  4165. u16 fop;
  4166. u64 rip;
  4167. u64 rdp;
  4168. u32 mxcsr;
  4169. u32 mxcsr_mask;
  4170. u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */
  4171. #ifdef CONFIG_X86_64
  4172. u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */
  4173. #else
  4174. u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */
  4175. #endif
  4176. };
  4177. /*
  4178. * Translate a guest virtual address to a guest physical address.
  4179. */
  4180. int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  4181. struct kvm_translation *tr)
  4182. {
  4183. unsigned long vaddr = tr->linear_address;
  4184. gpa_t gpa;
  4185. vcpu_load(vcpu);
  4186. down_read(&vcpu->kvm->slots_lock);
  4187. gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, vaddr);
  4188. up_read(&vcpu->kvm->slots_lock);
  4189. tr->physical_address = gpa;
  4190. tr->valid = gpa != UNMAPPED_GVA;
  4191. tr->writeable = 1;
  4192. tr->usermode = 0;
  4193. vcpu_put(vcpu);
  4194. return 0;
  4195. }
  4196. int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  4197. {
  4198. struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
  4199. vcpu_load(vcpu);
  4200. memcpy(fpu->fpr, fxsave->st_space, 128);
  4201. fpu->fcw = fxsave->cwd;
  4202. fpu->fsw = fxsave->swd;
  4203. fpu->ftwx = fxsave->twd;
  4204. fpu->last_opcode = fxsave->fop;
  4205. fpu->last_ip = fxsave->rip;
  4206. fpu->last_dp = fxsave->rdp;
  4207. memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
  4208. vcpu_put(vcpu);
  4209. return 0;
  4210. }
  4211. int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  4212. {
  4213. struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
  4214. vcpu_load(vcpu);
  4215. memcpy(fxsave->st_space, fpu->fpr, 128);
  4216. fxsave->cwd = fpu->fcw;
  4217. fxsave->swd = fpu->fsw;
  4218. fxsave->twd = fpu->ftwx;
  4219. fxsave->fop = fpu->last_opcode;
  4220. fxsave->rip = fpu->last_ip;
  4221. fxsave->rdp = fpu->last_dp;
  4222. memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
  4223. vcpu_put(vcpu);
  4224. return 0;
  4225. }
  4226. void fx_init(struct kvm_vcpu *vcpu)
  4227. {
  4228. unsigned after_mxcsr_mask;
  4229. /*
  4230. * Touch the fpu the first time in non atomic context as if
  4231. * this is the first fpu instruction the exception handler
  4232. * will fire before the instruction returns and it'll have to
  4233. * allocate ram with GFP_KERNEL.
  4234. */
  4235. if (!used_math())
  4236. kvm_fx_save(&vcpu->arch.host_fx_image);
  4237. /* Initialize guest FPU by resetting ours and saving into guest's */
  4238. preempt_disable();
  4239. kvm_fx_save(&vcpu->arch.host_fx_image);
  4240. kvm_fx_finit();
  4241. kvm_fx_save(&vcpu->arch.guest_fx_image);
  4242. kvm_fx_restore(&vcpu->arch.host_fx_image);
  4243. preempt_enable();
  4244. vcpu->arch.cr0 |= X86_CR0_ET;
  4245. after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
  4246. vcpu->arch.guest_fx_image.mxcsr = 0x1f80;
  4247. memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask,
  4248. 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
  4249. }
  4250. EXPORT_SYMBOL_GPL(fx_init);
  4251. void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
  4252. {
  4253. if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
  4254. return;
  4255. vcpu->guest_fpu_loaded = 1;
  4256. kvm_fx_save(&vcpu->arch.host_fx_image);
  4257. kvm_fx_restore(&vcpu->arch.guest_fx_image);
  4258. }
  4259. EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
  4260. void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
  4261. {
  4262. if (!vcpu->guest_fpu_loaded)
  4263. return;
  4264. vcpu->guest_fpu_loaded = 0;
  4265. kvm_fx_save(&vcpu->arch.guest_fx_image);
  4266. kvm_fx_restore(&vcpu->arch.host_fx_image);
  4267. ++vcpu->stat.fpu_reload;
  4268. }
  4269. EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
  4270. void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
  4271. {
  4272. if (vcpu->arch.time_page) {
  4273. kvm_release_page_dirty(vcpu->arch.time_page);
  4274. vcpu->arch.time_page = NULL;
  4275. }
  4276. kvm_x86_ops->vcpu_free(vcpu);
  4277. }
  4278. struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
  4279. unsigned int id)
  4280. {
  4281. return kvm_x86_ops->vcpu_create(kvm, id);
  4282. }
  4283. int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
  4284. {
  4285. int r;
  4286. /* We do fxsave: this must be aligned. */
  4287. BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF);
  4288. vcpu->arch.mtrr_state.have_fixed = 1;
  4289. vcpu_load(vcpu);
  4290. r = kvm_arch_vcpu_reset(vcpu);
  4291. if (r == 0)
  4292. r = kvm_mmu_setup(vcpu);
  4293. vcpu_put(vcpu);
  4294. if (r < 0)
  4295. goto free_vcpu;
  4296. return 0;
  4297. free_vcpu:
  4298. kvm_x86_ops->vcpu_free(vcpu);
  4299. return r;
  4300. }
  4301. void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
  4302. {
  4303. vcpu_load(vcpu);
  4304. kvm_mmu_unload(vcpu);
  4305. vcpu_put(vcpu);
  4306. kvm_x86_ops->vcpu_free(vcpu);
  4307. }
  4308. int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
  4309. {
  4310. vcpu->arch.nmi_pending = false;
  4311. vcpu->arch.nmi_injected = false;
  4312. vcpu->arch.switch_db_regs = 0;
  4313. memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
  4314. vcpu->arch.dr6 = DR6_FIXED_1;
  4315. vcpu->arch.dr7 = DR7_FIXED_1;
  4316. return kvm_x86_ops->vcpu_reset(vcpu);
  4317. }
  4318. int kvm_arch_hardware_enable(void *garbage)
  4319. {
  4320. /*
  4321. * Since this may be called from a hotplug notifcation,
  4322. * we can't get the CPU frequency directly.
  4323. */
  4324. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
  4325. int cpu = raw_smp_processor_id();
  4326. per_cpu(cpu_tsc_khz, cpu) = 0;
  4327. }
  4328. kvm_shared_msr_cpu_online();
  4329. return kvm_x86_ops->hardware_enable(garbage);
  4330. }
  4331. void kvm_arch_hardware_disable(void *garbage)
  4332. {
  4333. kvm_x86_ops->hardware_disable(garbage);
  4334. drop_user_return_notifiers(garbage);
  4335. }
  4336. int kvm_arch_hardware_setup(void)
  4337. {
  4338. return kvm_x86_ops->hardware_setup();
  4339. }
  4340. void kvm_arch_hardware_unsetup(void)
  4341. {
  4342. kvm_x86_ops->hardware_unsetup();
  4343. }
  4344. void kvm_arch_check_processor_compat(void *rtn)
  4345. {
  4346. kvm_x86_ops->check_processor_compatibility(rtn);
  4347. }
  4348. int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
  4349. {
  4350. struct page *page;
  4351. struct kvm *kvm;
  4352. int r;
  4353. BUG_ON(vcpu->kvm == NULL);
  4354. kvm = vcpu->kvm;
  4355. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  4356. if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
  4357. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  4358. else
  4359. vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
  4360. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  4361. if (!page) {
  4362. r = -ENOMEM;
  4363. goto fail;
  4364. }
  4365. vcpu->arch.pio_data = page_address(page);
  4366. r = kvm_mmu_create(vcpu);
  4367. if (r < 0)
  4368. goto fail_free_pio_data;
  4369. if (irqchip_in_kernel(kvm)) {
  4370. r = kvm_create_lapic(vcpu);
  4371. if (r < 0)
  4372. goto fail_mmu_destroy;
  4373. }
  4374. vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
  4375. GFP_KERNEL);
  4376. if (!vcpu->arch.mce_banks) {
  4377. r = -ENOMEM;
  4378. goto fail_free_lapic;
  4379. }
  4380. vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
  4381. return 0;
  4382. fail_free_lapic:
  4383. kvm_free_lapic(vcpu);
  4384. fail_mmu_destroy:
  4385. kvm_mmu_destroy(vcpu);
  4386. fail_free_pio_data:
  4387. free_page((unsigned long)vcpu->arch.pio_data);
  4388. fail:
  4389. return r;
  4390. }
  4391. void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
  4392. {
  4393. kfree(vcpu->arch.mce_banks);
  4394. kvm_free_lapic(vcpu);
  4395. down_read(&vcpu->kvm->slots_lock);
  4396. kvm_mmu_destroy(vcpu);
  4397. up_read(&vcpu->kvm->slots_lock);
  4398. free_page((unsigned long)vcpu->arch.pio_data);
  4399. }
  4400. struct kvm *kvm_arch_create_vm(void)
  4401. {
  4402. struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
  4403. if (!kvm)
  4404. return ERR_PTR(-ENOMEM);
  4405. INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
  4406. INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
  4407. /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
  4408. set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
  4409. rdtscll(kvm->arch.vm_init_tsc);
  4410. return kvm;
  4411. }
  4412. static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
  4413. {
  4414. vcpu_load(vcpu);
  4415. kvm_mmu_unload(vcpu);
  4416. vcpu_put(vcpu);
  4417. }
  4418. static void kvm_free_vcpus(struct kvm *kvm)
  4419. {
  4420. unsigned int i;
  4421. struct kvm_vcpu *vcpu;
  4422. /*
  4423. * Unpin any mmu pages first.
  4424. */
  4425. kvm_for_each_vcpu(i, vcpu, kvm)
  4426. kvm_unload_vcpu_mmu(vcpu);
  4427. kvm_for_each_vcpu(i, vcpu, kvm)
  4428. kvm_arch_vcpu_free(vcpu);
  4429. mutex_lock(&kvm->lock);
  4430. for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
  4431. kvm->vcpus[i] = NULL;
  4432. atomic_set(&kvm->online_vcpus, 0);
  4433. mutex_unlock(&kvm->lock);
  4434. }
  4435. void kvm_arch_sync_events(struct kvm *kvm)
  4436. {
  4437. kvm_free_all_assigned_devices(kvm);
  4438. }
  4439. void kvm_arch_destroy_vm(struct kvm *kvm)
  4440. {
  4441. kvm_iommu_unmap_guest(kvm);
  4442. kvm_free_pit(kvm);
  4443. kfree(kvm->arch.vpic);
  4444. kfree(kvm->arch.vioapic);
  4445. kvm_free_vcpus(kvm);
  4446. kvm_free_physmem(kvm);
  4447. if (kvm->arch.apic_access_page)
  4448. put_page(kvm->arch.apic_access_page);
  4449. if (kvm->arch.ept_identity_pagetable)
  4450. put_page(kvm->arch.ept_identity_pagetable);
  4451. kfree(kvm);
  4452. }
  4453. int kvm_arch_set_memory_region(struct kvm *kvm,
  4454. struct kvm_userspace_memory_region *mem,
  4455. struct kvm_memory_slot old,
  4456. int user_alloc)
  4457. {
  4458. int npages = mem->memory_size >> PAGE_SHIFT;
  4459. struct kvm_memory_slot *memslot = &kvm->memslots[mem->slot];
  4460. /*To keep backward compatibility with older userspace,
  4461. *x86 needs to hanlde !user_alloc case.
  4462. */
  4463. if (!user_alloc) {
  4464. if (npages && !old.rmap) {
  4465. unsigned long userspace_addr;
  4466. down_write(&current->mm->mmap_sem);
  4467. userspace_addr = do_mmap(NULL, 0,
  4468. npages * PAGE_SIZE,
  4469. PROT_READ | PROT_WRITE,
  4470. MAP_PRIVATE | MAP_ANONYMOUS,
  4471. 0);
  4472. up_write(&current->mm->mmap_sem);
  4473. if (IS_ERR((void *)userspace_addr))
  4474. return PTR_ERR((void *)userspace_addr);
  4475. /* set userspace_addr atomically for kvm_hva_to_rmapp */
  4476. spin_lock(&kvm->mmu_lock);
  4477. memslot->userspace_addr = userspace_addr;
  4478. spin_unlock(&kvm->mmu_lock);
  4479. } else {
  4480. if (!old.user_alloc && old.rmap) {
  4481. int ret;
  4482. down_write(&current->mm->mmap_sem);
  4483. ret = do_munmap(current->mm, old.userspace_addr,
  4484. old.npages * PAGE_SIZE);
  4485. up_write(&current->mm->mmap_sem);
  4486. if (ret < 0)
  4487. printk(KERN_WARNING
  4488. "kvm_vm_ioctl_set_memory_region: "
  4489. "failed to munmap memory\n");
  4490. }
  4491. }
  4492. }
  4493. spin_lock(&kvm->mmu_lock);
  4494. if (!kvm->arch.n_requested_mmu_pages) {
  4495. unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
  4496. kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
  4497. }
  4498. kvm_mmu_slot_remove_write_access(kvm, mem->slot);
  4499. spin_unlock(&kvm->mmu_lock);
  4500. return 0;
  4501. }
  4502. void kvm_arch_flush_shadow(struct kvm *kvm)
  4503. {
  4504. kvm_mmu_zap_all(kvm);
  4505. kvm_reload_remote_mmus(kvm);
  4506. }
  4507. int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
  4508. {
  4509. return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
  4510. || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
  4511. || vcpu->arch.nmi_pending ||
  4512. (kvm_arch_interrupt_allowed(vcpu) &&
  4513. kvm_cpu_has_interrupt(vcpu));
  4514. }
  4515. void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
  4516. {
  4517. int me;
  4518. int cpu = vcpu->cpu;
  4519. if (waitqueue_active(&vcpu->wq)) {
  4520. wake_up_interruptible(&vcpu->wq);
  4521. ++vcpu->stat.halt_wakeup;
  4522. }
  4523. me = get_cpu();
  4524. if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
  4525. if (!test_and_set_bit(KVM_REQ_KICK, &vcpu->requests))
  4526. smp_send_reschedule(cpu);
  4527. put_cpu();
  4528. }
  4529. int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
  4530. {
  4531. return kvm_x86_ops->interrupt_allowed(vcpu);
  4532. }
  4533. unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
  4534. {
  4535. unsigned long rflags;
  4536. rflags = kvm_x86_ops->get_rflags(vcpu);
  4537. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
  4538. rflags &= ~(unsigned long)(X86_EFLAGS_TF | X86_EFLAGS_RF);
  4539. return rflags;
  4540. }
  4541. EXPORT_SYMBOL_GPL(kvm_get_rflags);
  4542. void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  4543. {
  4544. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
  4545. vcpu->arch.singlestep_cs ==
  4546. get_segment_selector(vcpu, VCPU_SREG_CS) &&
  4547. vcpu->arch.singlestep_rip == kvm_rip_read(vcpu))
  4548. rflags |= X86_EFLAGS_TF | X86_EFLAGS_RF;
  4549. kvm_x86_ops->set_rflags(vcpu, rflags);
  4550. }
  4551. EXPORT_SYMBOL_GPL(kvm_set_rflags);
  4552. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
  4553. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
  4554. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
  4555. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
  4556. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
  4557. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
  4558. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
  4559. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
  4560. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
  4561. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
  4562. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);