x86.c 132 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * derived from drivers/kvm/kvm_main.c
  5. *
  6. * Copyright (C) 2006 Qumranet, Inc.
  7. * Copyright (C) 2008 Qumranet, Inc.
  8. * Copyright IBM Corporation, 2008
  9. *
  10. * Authors:
  11. * Avi Kivity <avi@qumranet.com>
  12. * Yaniv Kamay <yaniv@qumranet.com>
  13. * Amit Shah <amit.shah@qumranet.com>
  14. * Ben-Ami Yassour <benami@il.ibm.com>
  15. *
  16. * This work is licensed under the terms of the GNU GPL, version 2. See
  17. * the COPYING file in the top-level directory.
  18. *
  19. */
  20. #include <linux/kvm_host.h>
  21. #include "irq.h"
  22. #include "mmu.h"
  23. #include "i8254.h"
  24. #include "tss.h"
  25. #include "kvm_cache_regs.h"
  26. #include "x86.h"
  27. #include <linux/clocksource.h>
  28. #include <linux/interrupt.h>
  29. #include <linux/kvm.h>
  30. #include <linux/fs.h>
  31. #include <linux/vmalloc.h>
  32. #include <linux/module.h>
  33. #include <linux/mman.h>
  34. #include <linux/highmem.h>
  35. #include <linux/iommu.h>
  36. #include <linux/intel-iommu.h>
  37. #include <linux/cpufreq.h>
  38. #include <linux/user-return-notifier.h>
  39. #include <linux/srcu.h>
  40. #include <trace/events/kvm.h>
  41. #undef TRACE_INCLUDE_FILE
  42. #define CREATE_TRACE_POINTS
  43. #include "trace.h"
  44. #include <asm/debugreg.h>
  45. #include <asm/uaccess.h>
  46. #include <asm/msr.h>
  47. #include <asm/desc.h>
  48. #include <asm/mtrr.h>
  49. #include <asm/mce.h>
  50. #define MAX_IO_MSRS 256
  51. #define CR0_RESERVED_BITS \
  52. (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
  53. | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
  54. | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
  55. #define CR4_RESERVED_BITS \
  56. (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
  57. | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
  58. | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \
  59. | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
  60. #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
  61. #define KVM_MAX_MCE_BANKS 32
  62. #define KVM_MCE_CAP_SUPPORTED MCG_CTL_P
  63. /* EFER defaults:
  64. * - enable syscall per default because its emulated by KVM
  65. * - enable LME and LMA per default on 64 bit KVM
  66. */
  67. #ifdef CONFIG_X86_64
  68. static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffafeULL;
  69. #else
  70. static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffffeULL;
  71. #endif
  72. #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  73. #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  74. static void update_cr8_intercept(struct kvm_vcpu *vcpu);
  75. static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  76. struct kvm_cpuid_entry2 __user *entries);
  77. struct kvm_x86_ops *kvm_x86_ops;
  78. EXPORT_SYMBOL_GPL(kvm_x86_ops);
  79. int ignore_msrs = 0;
  80. module_param_named(ignore_msrs, ignore_msrs, bool, S_IRUGO | S_IWUSR);
  81. #define KVM_NR_SHARED_MSRS 16
  82. struct kvm_shared_msrs_global {
  83. int nr;
  84. u32 msrs[KVM_NR_SHARED_MSRS];
  85. };
  86. struct kvm_shared_msrs {
  87. struct user_return_notifier urn;
  88. bool registered;
  89. struct kvm_shared_msr_values {
  90. u64 host;
  91. u64 curr;
  92. } values[KVM_NR_SHARED_MSRS];
  93. };
  94. static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
  95. static DEFINE_PER_CPU(struct kvm_shared_msrs, shared_msrs);
  96. struct kvm_stats_debugfs_item debugfs_entries[] = {
  97. { "pf_fixed", VCPU_STAT(pf_fixed) },
  98. { "pf_guest", VCPU_STAT(pf_guest) },
  99. { "tlb_flush", VCPU_STAT(tlb_flush) },
  100. { "invlpg", VCPU_STAT(invlpg) },
  101. { "exits", VCPU_STAT(exits) },
  102. { "io_exits", VCPU_STAT(io_exits) },
  103. { "mmio_exits", VCPU_STAT(mmio_exits) },
  104. { "signal_exits", VCPU_STAT(signal_exits) },
  105. { "irq_window", VCPU_STAT(irq_window_exits) },
  106. { "nmi_window", VCPU_STAT(nmi_window_exits) },
  107. { "halt_exits", VCPU_STAT(halt_exits) },
  108. { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  109. { "hypercalls", VCPU_STAT(hypercalls) },
  110. { "request_irq", VCPU_STAT(request_irq_exits) },
  111. { "irq_exits", VCPU_STAT(irq_exits) },
  112. { "host_state_reload", VCPU_STAT(host_state_reload) },
  113. { "efer_reload", VCPU_STAT(efer_reload) },
  114. { "fpu_reload", VCPU_STAT(fpu_reload) },
  115. { "insn_emulation", VCPU_STAT(insn_emulation) },
  116. { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
  117. { "irq_injections", VCPU_STAT(irq_injections) },
  118. { "nmi_injections", VCPU_STAT(nmi_injections) },
  119. { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
  120. { "mmu_pte_write", VM_STAT(mmu_pte_write) },
  121. { "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
  122. { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
  123. { "mmu_flooded", VM_STAT(mmu_flooded) },
  124. { "mmu_recycled", VM_STAT(mmu_recycled) },
  125. { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
  126. { "mmu_unsync", VM_STAT(mmu_unsync) },
  127. { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
  128. { "largepages", VM_STAT(lpages) },
  129. { NULL }
  130. };
  131. static void kvm_on_user_return(struct user_return_notifier *urn)
  132. {
  133. unsigned slot;
  134. struct kvm_shared_msrs *locals
  135. = container_of(urn, struct kvm_shared_msrs, urn);
  136. struct kvm_shared_msr_values *values;
  137. for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
  138. values = &locals->values[slot];
  139. if (values->host != values->curr) {
  140. wrmsrl(shared_msrs_global.msrs[slot], values->host);
  141. values->curr = values->host;
  142. }
  143. }
  144. locals->registered = false;
  145. user_return_notifier_unregister(urn);
  146. }
  147. static void shared_msr_update(unsigned slot, u32 msr)
  148. {
  149. struct kvm_shared_msrs *smsr;
  150. u64 value;
  151. smsr = &__get_cpu_var(shared_msrs);
  152. /* only read, and nobody should modify it at this time,
  153. * so don't need lock */
  154. if (slot >= shared_msrs_global.nr) {
  155. printk(KERN_ERR "kvm: invalid MSR slot!");
  156. return;
  157. }
  158. rdmsrl_safe(msr, &value);
  159. smsr->values[slot].host = value;
  160. smsr->values[slot].curr = value;
  161. }
  162. void kvm_define_shared_msr(unsigned slot, u32 msr)
  163. {
  164. if (slot >= shared_msrs_global.nr)
  165. shared_msrs_global.nr = slot + 1;
  166. shared_msrs_global.msrs[slot] = msr;
  167. /* we need ensured the shared_msr_global have been updated */
  168. smp_wmb();
  169. }
  170. EXPORT_SYMBOL_GPL(kvm_define_shared_msr);
  171. static void kvm_shared_msr_cpu_online(void)
  172. {
  173. unsigned i;
  174. for (i = 0; i < shared_msrs_global.nr; ++i)
  175. shared_msr_update(i, shared_msrs_global.msrs[i]);
  176. }
  177. void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
  178. {
  179. struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);
  180. if (((value ^ smsr->values[slot].curr) & mask) == 0)
  181. return;
  182. smsr->values[slot].curr = value;
  183. wrmsrl(shared_msrs_global.msrs[slot], value);
  184. if (!smsr->registered) {
  185. smsr->urn.on_user_return = kvm_on_user_return;
  186. user_return_notifier_register(&smsr->urn);
  187. smsr->registered = true;
  188. }
  189. }
  190. EXPORT_SYMBOL_GPL(kvm_set_shared_msr);
  191. static void drop_user_return_notifiers(void *ignore)
  192. {
  193. struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);
  194. if (smsr->registered)
  195. kvm_on_user_return(&smsr->urn);
  196. }
  197. unsigned long segment_base(u16 selector)
  198. {
  199. struct descriptor_table gdt;
  200. struct desc_struct *d;
  201. unsigned long table_base;
  202. unsigned long v;
  203. if (selector == 0)
  204. return 0;
  205. kvm_get_gdt(&gdt);
  206. table_base = gdt.base;
  207. if (selector & 4) { /* from ldt */
  208. u16 ldt_selector = kvm_read_ldt();
  209. table_base = segment_base(ldt_selector);
  210. }
  211. d = (struct desc_struct *)(table_base + (selector & ~7));
  212. v = get_desc_base(d);
  213. #ifdef CONFIG_X86_64
  214. if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
  215. v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
  216. #endif
  217. return v;
  218. }
  219. EXPORT_SYMBOL_GPL(segment_base);
  220. u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
  221. {
  222. if (irqchip_in_kernel(vcpu->kvm))
  223. return vcpu->arch.apic_base;
  224. else
  225. return vcpu->arch.apic_base;
  226. }
  227. EXPORT_SYMBOL_GPL(kvm_get_apic_base);
  228. void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
  229. {
  230. /* TODO: reserve bits check */
  231. if (irqchip_in_kernel(vcpu->kvm))
  232. kvm_lapic_set_base(vcpu, data);
  233. else
  234. vcpu->arch.apic_base = data;
  235. }
  236. EXPORT_SYMBOL_GPL(kvm_set_apic_base);
  237. #define EXCPT_BENIGN 0
  238. #define EXCPT_CONTRIBUTORY 1
  239. #define EXCPT_PF 2
  240. static int exception_class(int vector)
  241. {
  242. switch (vector) {
  243. case PF_VECTOR:
  244. return EXCPT_PF;
  245. case DE_VECTOR:
  246. case TS_VECTOR:
  247. case NP_VECTOR:
  248. case SS_VECTOR:
  249. case GP_VECTOR:
  250. return EXCPT_CONTRIBUTORY;
  251. default:
  252. break;
  253. }
  254. return EXCPT_BENIGN;
  255. }
  256. static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
  257. unsigned nr, bool has_error, u32 error_code)
  258. {
  259. u32 prev_nr;
  260. int class1, class2;
  261. if (!vcpu->arch.exception.pending) {
  262. queue:
  263. vcpu->arch.exception.pending = true;
  264. vcpu->arch.exception.has_error_code = has_error;
  265. vcpu->arch.exception.nr = nr;
  266. vcpu->arch.exception.error_code = error_code;
  267. return;
  268. }
  269. /* to check exception */
  270. prev_nr = vcpu->arch.exception.nr;
  271. if (prev_nr == DF_VECTOR) {
  272. /* triple fault -> shutdown */
  273. set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
  274. return;
  275. }
  276. class1 = exception_class(prev_nr);
  277. class2 = exception_class(nr);
  278. if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
  279. || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
  280. /* generate double fault per SDM Table 5-5 */
  281. vcpu->arch.exception.pending = true;
  282. vcpu->arch.exception.has_error_code = true;
  283. vcpu->arch.exception.nr = DF_VECTOR;
  284. vcpu->arch.exception.error_code = 0;
  285. } else
  286. /* replace previous exception with a new one in a hope
  287. that instruction re-execution will regenerate lost
  288. exception */
  289. goto queue;
  290. }
  291. void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
  292. {
  293. kvm_multiple_exception(vcpu, nr, false, 0);
  294. }
  295. EXPORT_SYMBOL_GPL(kvm_queue_exception);
  296. void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr,
  297. u32 error_code)
  298. {
  299. ++vcpu->stat.pf_guest;
  300. vcpu->arch.cr2 = addr;
  301. kvm_queue_exception_e(vcpu, PF_VECTOR, error_code);
  302. }
  303. void kvm_inject_nmi(struct kvm_vcpu *vcpu)
  304. {
  305. vcpu->arch.nmi_pending = 1;
  306. }
  307. EXPORT_SYMBOL_GPL(kvm_inject_nmi);
  308. void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
  309. {
  310. kvm_multiple_exception(vcpu, nr, true, error_code);
  311. }
  312. EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
  313. /*
  314. * Checks if cpl <= required_cpl; if true, return true. Otherwise queue
  315. * a #GP and return false.
  316. */
  317. bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
  318. {
  319. if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
  320. return true;
  321. kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
  322. return false;
  323. }
  324. EXPORT_SYMBOL_GPL(kvm_require_cpl);
  325. /*
  326. * Load the pae pdptrs. Return true is they are all valid.
  327. */
  328. int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
  329. {
  330. gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
  331. unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
  332. int i;
  333. int ret;
  334. u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
  335. ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte,
  336. offset * sizeof(u64), sizeof(pdpte));
  337. if (ret < 0) {
  338. ret = 0;
  339. goto out;
  340. }
  341. for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
  342. if (is_present_gpte(pdpte[i]) &&
  343. (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
  344. ret = 0;
  345. goto out;
  346. }
  347. }
  348. ret = 1;
  349. memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
  350. __set_bit(VCPU_EXREG_PDPTR,
  351. (unsigned long *)&vcpu->arch.regs_avail);
  352. __set_bit(VCPU_EXREG_PDPTR,
  353. (unsigned long *)&vcpu->arch.regs_dirty);
  354. out:
  355. return ret;
  356. }
  357. EXPORT_SYMBOL_GPL(load_pdptrs);
  358. static bool pdptrs_changed(struct kvm_vcpu *vcpu)
  359. {
  360. u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
  361. bool changed = true;
  362. int r;
  363. if (is_long_mode(vcpu) || !is_pae(vcpu))
  364. return false;
  365. if (!test_bit(VCPU_EXREG_PDPTR,
  366. (unsigned long *)&vcpu->arch.regs_avail))
  367. return true;
  368. r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte));
  369. if (r < 0)
  370. goto out;
  371. changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0;
  372. out:
  373. return changed;
  374. }
  375. void kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  376. {
  377. if (cr0 & CR0_RESERVED_BITS) {
  378. printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
  379. cr0, vcpu->arch.cr0);
  380. kvm_inject_gp(vcpu, 0);
  381. return;
  382. }
  383. if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
  384. printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
  385. kvm_inject_gp(vcpu, 0);
  386. return;
  387. }
  388. if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
  389. printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
  390. "and a clear PE flag\n");
  391. kvm_inject_gp(vcpu, 0);
  392. return;
  393. }
  394. if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
  395. #ifdef CONFIG_X86_64
  396. if ((vcpu->arch.shadow_efer & EFER_LME)) {
  397. int cs_db, cs_l;
  398. if (!is_pae(vcpu)) {
  399. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  400. "in long mode while PAE is disabled\n");
  401. kvm_inject_gp(vcpu, 0);
  402. return;
  403. }
  404. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  405. if (cs_l) {
  406. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  407. "in long mode while CS.L == 1\n");
  408. kvm_inject_gp(vcpu, 0);
  409. return;
  410. }
  411. } else
  412. #endif
  413. if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
  414. printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
  415. "reserved bits\n");
  416. kvm_inject_gp(vcpu, 0);
  417. return;
  418. }
  419. }
  420. kvm_x86_ops->set_cr0(vcpu, cr0);
  421. vcpu->arch.cr0 = cr0;
  422. kvm_mmu_reset_context(vcpu);
  423. return;
  424. }
  425. EXPORT_SYMBOL_GPL(kvm_set_cr0);
  426. void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
  427. {
  428. kvm_set_cr0(vcpu, (vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f));
  429. }
  430. EXPORT_SYMBOL_GPL(kvm_lmsw);
  431. void kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  432. {
  433. unsigned long old_cr4 = kvm_read_cr4(vcpu);
  434. unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE;
  435. if (cr4 & CR4_RESERVED_BITS) {
  436. printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
  437. kvm_inject_gp(vcpu, 0);
  438. return;
  439. }
  440. if (is_long_mode(vcpu)) {
  441. if (!(cr4 & X86_CR4_PAE)) {
  442. printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
  443. "in long mode\n");
  444. kvm_inject_gp(vcpu, 0);
  445. return;
  446. }
  447. } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
  448. && ((cr4 ^ old_cr4) & pdptr_bits)
  449. && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
  450. printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
  451. kvm_inject_gp(vcpu, 0);
  452. return;
  453. }
  454. if (cr4 & X86_CR4_VMXE) {
  455. printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
  456. kvm_inject_gp(vcpu, 0);
  457. return;
  458. }
  459. kvm_x86_ops->set_cr4(vcpu, cr4);
  460. vcpu->arch.cr4 = cr4;
  461. vcpu->arch.mmu.base_role.cr4_pge = (cr4 & X86_CR4_PGE) && !tdp_enabled;
  462. kvm_mmu_reset_context(vcpu);
  463. }
  464. EXPORT_SYMBOL_GPL(kvm_set_cr4);
  465. void kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
  466. {
  467. if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) {
  468. kvm_mmu_sync_roots(vcpu);
  469. kvm_mmu_flush_tlb(vcpu);
  470. return;
  471. }
  472. if (is_long_mode(vcpu)) {
  473. if (cr3 & CR3_L_MODE_RESERVED_BITS) {
  474. printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
  475. kvm_inject_gp(vcpu, 0);
  476. return;
  477. }
  478. } else {
  479. if (is_pae(vcpu)) {
  480. if (cr3 & CR3_PAE_RESERVED_BITS) {
  481. printk(KERN_DEBUG
  482. "set_cr3: #GP, reserved bits\n");
  483. kvm_inject_gp(vcpu, 0);
  484. return;
  485. }
  486. if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
  487. printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
  488. "reserved bits\n");
  489. kvm_inject_gp(vcpu, 0);
  490. return;
  491. }
  492. }
  493. /*
  494. * We don't check reserved bits in nonpae mode, because
  495. * this isn't enforced, and VMware depends on this.
  496. */
  497. }
  498. /*
  499. * Does the new cr3 value map to physical memory? (Note, we
  500. * catch an invalid cr3 even in real-mode, because it would
  501. * cause trouble later on when we turn on paging anyway.)
  502. *
  503. * A real CPU would silently accept an invalid cr3 and would
  504. * attempt to use it - with largely undefined (and often hard
  505. * to debug) behavior on the guest side.
  506. */
  507. if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
  508. kvm_inject_gp(vcpu, 0);
  509. else {
  510. vcpu->arch.cr3 = cr3;
  511. vcpu->arch.mmu.new_cr3(vcpu);
  512. }
  513. }
  514. EXPORT_SYMBOL_GPL(kvm_set_cr3);
  515. void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
  516. {
  517. if (cr8 & CR8_RESERVED_BITS) {
  518. printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
  519. kvm_inject_gp(vcpu, 0);
  520. return;
  521. }
  522. if (irqchip_in_kernel(vcpu->kvm))
  523. kvm_lapic_set_tpr(vcpu, cr8);
  524. else
  525. vcpu->arch.cr8 = cr8;
  526. }
  527. EXPORT_SYMBOL_GPL(kvm_set_cr8);
  528. unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
  529. {
  530. if (irqchip_in_kernel(vcpu->kvm))
  531. return kvm_lapic_get_cr8(vcpu);
  532. else
  533. return vcpu->arch.cr8;
  534. }
  535. EXPORT_SYMBOL_GPL(kvm_get_cr8);
  536. static inline u32 bit(int bitno)
  537. {
  538. return 1 << (bitno & 31);
  539. }
  540. /*
  541. * List of msr numbers which we expose to userspace through KVM_GET_MSRS
  542. * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
  543. *
  544. * This list is modified at module load time to reflect the
  545. * capabilities of the host cpu. This capabilities test skips MSRs that are
  546. * kvm-specific. Those are put in the beginning of the list.
  547. */
  548. #define KVM_SAVE_MSRS_BEGIN 2
  549. static u32 msrs_to_save[] = {
  550. MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
  551. MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  552. MSR_K6_STAR,
  553. #ifdef CONFIG_X86_64
  554. MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
  555. #endif
  556. MSR_IA32_TSC, MSR_IA32_PERF_STATUS, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA
  557. };
  558. static unsigned num_msrs_to_save;
  559. static u32 emulated_msrs[] = {
  560. MSR_IA32_MISC_ENABLE,
  561. };
  562. static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
  563. {
  564. if (efer & efer_reserved_bits) {
  565. printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
  566. efer);
  567. kvm_inject_gp(vcpu, 0);
  568. return;
  569. }
  570. if (is_paging(vcpu)
  571. && (vcpu->arch.shadow_efer & EFER_LME) != (efer & EFER_LME)) {
  572. printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
  573. kvm_inject_gp(vcpu, 0);
  574. return;
  575. }
  576. if (efer & EFER_FFXSR) {
  577. struct kvm_cpuid_entry2 *feat;
  578. feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
  579. if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT))) {
  580. printk(KERN_DEBUG "set_efer: #GP, enable FFXSR w/o CPUID capability\n");
  581. kvm_inject_gp(vcpu, 0);
  582. return;
  583. }
  584. }
  585. if (efer & EFER_SVME) {
  586. struct kvm_cpuid_entry2 *feat;
  587. feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
  588. if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM))) {
  589. printk(KERN_DEBUG "set_efer: #GP, enable SVM w/o SVM\n");
  590. kvm_inject_gp(vcpu, 0);
  591. return;
  592. }
  593. }
  594. kvm_x86_ops->set_efer(vcpu, efer);
  595. efer &= ~EFER_LMA;
  596. efer |= vcpu->arch.shadow_efer & EFER_LMA;
  597. vcpu->arch.shadow_efer = efer;
  598. vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
  599. kvm_mmu_reset_context(vcpu);
  600. }
  601. void kvm_enable_efer_bits(u64 mask)
  602. {
  603. efer_reserved_bits &= ~mask;
  604. }
  605. EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
  606. /*
  607. * Writes msr value into into the appropriate "register".
  608. * Returns 0 on success, non-0 otherwise.
  609. * Assumes vcpu_load() was already called.
  610. */
  611. int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
  612. {
  613. return kvm_x86_ops->set_msr(vcpu, msr_index, data);
  614. }
  615. /*
  616. * Adapt set_msr() to msr_io()'s calling convention
  617. */
  618. static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
  619. {
  620. return kvm_set_msr(vcpu, index, *data);
  621. }
  622. static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
  623. {
  624. static int version;
  625. struct pvclock_wall_clock wc;
  626. struct timespec boot;
  627. if (!wall_clock)
  628. return;
  629. version++;
  630. kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
  631. /*
  632. * The guest calculates current wall clock time by adding
  633. * system time (updated by kvm_write_guest_time below) to the
  634. * wall clock specified here. guest system time equals host
  635. * system time for us, thus we must fill in host boot time here.
  636. */
  637. getboottime(&boot);
  638. wc.sec = boot.tv_sec;
  639. wc.nsec = boot.tv_nsec;
  640. wc.version = version;
  641. kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
  642. version++;
  643. kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
  644. }
  645. static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
  646. {
  647. uint32_t quotient, remainder;
  648. /* Don't try to replace with do_div(), this one calculates
  649. * "(dividend << 32) / divisor" */
  650. __asm__ ( "divl %4"
  651. : "=a" (quotient), "=d" (remainder)
  652. : "0" (0), "1" (dividend), "r" (divisor) );
  653. return quotient;
  654. }
  655. static void kvm_set_time_scale(uint32_t tsc_khz, struct pvclock_vcpu_time_info *hv_clock)
  656. {
  657. uint64_t nsecs = 1000000000LL;
  658. int32_t shift = 0;
  659. uint64_t tps64;
  660. uint32_t tps32;
  661. tps64 = tsc_khz * 1000LL;
  662. while (tps64 > nsecs*2) {
  663. tps64 >>= 1;
  664. shift--;
  665. }
  666. tps32 = (uint32_t)tps64;
  667. while (tps32 <= (uint32_t)nsecs) {
  668. tps32 <<= 1;
  669. shift++;
  670. }
  671. hv_clock->tsc_shift = shift;
  672. hv_clock->tsc_to_system_mul = div_frac(nsecs, tps32);
  673. pr_debug("%s: tsc_khz %u, tsc_shift %d, tsc_mul %u\n",
  674. __func__, tsc_khz, hv_clock->tsc_shift,
  675. hv_clock->tsc_to_system_mul);
  676. }
  677. static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
  678. static void kvm_write_guest_time(struct kvm_vcpu *v)
  679. {
  680. struct timespec ts;
  681. unsigned long flags;
  682. struct kvm_vcpu_arch *vcpu = &v->arch;
  683. void *shared_kaddr;
  684. unsigned long this_tsc_khz;
  685. if ((!vcpu->time_page))
  686. return;
  687. this_tsc_khz = get_cpu_var(cpu_tsc_khz);
  688. if (unlikely(vcpu->hv_clock_tsc_khz != this_tsc_khz)) {
  689. kvm_set_time_scale(this_tsc_khz, &vcpu->hv_clock);
  690. vcpu->hv_clock_tsc_khz = this_tsc_khz;
  691. }
  692. put_cpu_var(cpu_tsc_khz);
  693. /* Keep irq disabled to prevent changes to the clock */
  694. local_irq_save(flags);
  695. kvm_get_msr(v, MSR_IA32_TSC, &vcpu->hv_clock.tsc_timestamp);
  696. ktime_get_ts(&ts);
  697. monotonic_to_bootbased(&ts);
  698. local_irq_restore(flags);
  699. /* With all the info we got, fill in the values */
  700. vcpu->hv_clock.system_time = ts.tv_nsec +
  701. (NSEC_PER_SEC * (u64)ts.tv_sec) + v->kvm->arch.kvmclock_offset;
  702. /*
  703. * The interface expects us to write an even number signaling that the
  704. * update is finished. Since the guest won't see the intermediate
  705. * state, we just increase by 2 at the end.
  706. */
  707. vcpu->hv_clock.version += 2;
  708. shared_kaddr = kmap_atomic(vcpu->time_page, KM_USER0);
  709. memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock,
  710. sizeof(vcpu->hv_clock));
  711. kunmap_atomic(shared_kaddr, KM_USER0);
  712. mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT);
  713. }
  714. static int kvm_request_guest_time_update(struct kvm_vcpu *v)
  715. {
  716. struct kvm_vcpu_arch *vcpu = &v->arch;
  717. if (!vcpu->time_page)
  718. return 0;
  719. set_bit(KVM_REQ_KVMCLOCK_UPDATE, &v->requests);
  720. return 1;
  721. }
  722. static bool msr_mtrr_valid(unsigned msr)
  723. {
  724. switch (msr) {
  725. case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
  726. case MSR_MTRRfix64K_00000:
  727. case MSR_MTRRfix16K_80000:
  728. case MSR_MTRRfix16K_A0000:
  729. case MSR_MTRRfix4K_C0000:
  730. case MSR_MTRRfix4K_C8000:
  731. case MSR_MTRRfix4K_D0000:
  732. case MSR_MTRRfix4K_D8000:
  733. case MSR_MTRRfix4K_E0000:
  734. case MSR_MTRRfix4K_E8000:
  735. case MSR_MTRRfix4K_F0000:
  736. case MSR_MTRRfix4K_F8000:
  737. case MSR_MTRRdefType:
  738. case MSR_IA32_CR_PAT:
  739. return true;
  740. case 0x2f8:
  741. return true;
  742. }
  743. return false;
  744. }
  745. static bool valid_pat_type(unsigned t)
  746. {
  747. return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
  748. }
  749. static bool valid_mtrr_type(unsigned t)
  750. {
  751. return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
  752. }
  753. static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  754. {
  755. int i;
  756. if (!msr_mtrr_valid(msr))
  757. return false;
  758. if (msr == MSR_IA32_CR_PAT) {
  759. for (i = 0; i < 8; i++)
  760. if (!valid_pat_type((data >> (i * 8)) & 0xff))
  761. return false;
  762. return true;
  763. } else if (msr == MSR_MTRRdefType) {
  764. if (data & ~0xcff)
  765. return false;
  766. return valid_mtrr_type(data & 0xff);
  767. } else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
  768. for (i = 0; i < 8 ; i++)
  769. if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
  770. return false;
  771. return true;
  772. }
  773. /* variable MTRRs */
  774. return valid_mtrr_type(data & 0xff);
  775. }
  776. static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  777. {
  778. u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
  779. if (!mtrr_valid(vcpu, msr, data))
  780. return 1;
  781. if (msr == MSR_MTRRdefType) {
  782. vcpu->arch.mtrr_state.def_type = data;
  783. vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
  784. } else if (msr == MSR_MTRRfix64K_00000)
  785. p[0] = data;
  786. else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
  787. p[1 + msr - MSR_MTRRfix16K_80000] = data;
  788. else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
  789. p[3 + msr - MSR_MTRRfix4K_C0000] = data;
  790. else if (msr == MSR_IA32_CR_PAT)
  791. vcpu->arch.pat = data;
  792. else { /* Variable MTRRs */
  793. int idx, is_mtrr_mask;
  794. u64 *pt;
  795. idx = (msr - 0x200) / 2;
  796. is_mtrr_mask = msr - 0x200 - 2 * idx;
  797. if (!is_mtrr_mask)
  798. pt =
  799. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
  800. else
  801. pt =
  802. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
  803. *pt = data;
  804. }
  805. kvm_mmu_reset_context(vcpu);
  806. return 0;
  807. }
  808. static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  809. {
  810. u64 mcg_cap = vcpu->arch.mcg_cap;
  811. unsigned bank_num = mcg_cap & 0xff;
  812. switch (msr) {
  813. case MSR_IA32_MCG_STATUS:
  814. vcpu->arch.mcg_status = data;
  815. break;
  816. case MSR_IA32_MCG_CTL:
  817. if (!(mcg_cap & MCG_CTL_P))
  818. return 1;
  819. if (data != 0 && data != ~(u64)0)
  820. return -1;
  821. vcpu->arch.mcg_ctl = data;
  822. break;
  823. default:
  824. if (msr >= MSR_IA32_MC0_CTL &&
  825. msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
  826. u32 offset = msr - MSR_IA32_MC0_CTL;
  827. /* only 0 or all 1s can be written to IA32_MCi_CTL */
  828. if ((offset & 0x3) == 0 &&
  829. data != 0 && data != ~(u64)0)
  830. return -1;
  831. vcpu->arch.mce_banks[offset] = data;
  832. break;
  833. }
  834. return 1;
  835. }
  836. return 0;
  837. }
  838. static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
  839. {
  840. struct kvm *kvm = vcpu->kvm;
  841. int lm = is_long_mode(vcpu);
  842. u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
  843. : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
  844. u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
  845. : kvm->arch.xen_hvm_config.blob_size_32;
  846. u32 page_num = data & ~PAGE_MASK;
  847. u64 page_addr = data & PAGE_MASK;
  848. u8 *page;
  849. int r;
  850. r = -E2BIG;
  851. if (page_num >= blob_size)
  852. goto out;
  853. r = -ENOMEM;
  854. page = kzalloc(PAGE_SIZE, GFP_KERNEL);
  855. if (!page)
  856. goto out;
  857. r = -EFAULT;
  858. if (copy_from_user(page, blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE))
  859. goto out_free;
  860. if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
  861. goto out_free;
  862. r = 0;
  863. out_free:
  864. kfree(page);
  865. out:
  866. return r;
  867. }
  868. int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  869. {
  870. switch (msr) {
  871. case MSR_EFER:
  872. set_efer(vcpu, data);
  873. break;
  874. case MSR_K7_HWCR:
  875. data &= ~(u64)0x40; /* ignore flush filter disable */
  876. if (data != 0) {
  877. pr_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
  878. data);
  879. return 1;
  880. }
  881. break;
  882. case MSR_FAM10H_MMIO_CONF_BASE:
  883. if (data != 0) {
  884. pr_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
  885. "0x%llx\n", data);
  886. return 1;
  887. }
  888. break;
  889. case MSR_AMD64_NB_CFG:
  890. break;
  891. case MSR_IA32_DEBUGCTLMSR:
  892. if (!data) {
  893. /* We support the non-activated case already */
  894. break;
  895. } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
  896. /* Values other than LBR and BTF are vendor-specific,
  897. thus reserved and should throw a #GP */
  898. return 1;
  899. }
  900. pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
  901. __func__, data);
  902. break;
  903. case MSR_IA32_UCODE_REV:
  904. case MSR_IA32_UCODE_WRITE:
  905. case MSR_VM_HSAVE_PA:
  906. case MSR_AMD64_PATCH_LOADER:
  907. break;
  908. case 0x200 ... 0x2ff:
  909. return set_msr_mtrr(vcpu, msr, data);
  910. case MSR_IA32_APICBASE:
  911. kvm_set_apic_base(vcpu, data);
  912. break;
  913. case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
  914. return kvm_x2apic_msr_write(vcpu, msr, data);
  915. case MSR_IA32_MISC_ENABLE:
  916. vcpu->arch.ia32_misc_enable_msr = data;
  917. break;
  918. case MSR_KVM_WALL_CLOCK:
  919. vcpu->kvm->arch.wall_clock = data;
  920. kvm_write_wall_clock(vcpu->kvm, data);
  921. break;
  922. case MSR_KVM_SYSTEM_TIME: {
  923. if (vcpu->arch.time_page) {
  924. kvm_release_page_dirty(vcpu->arch.time_page);
  925. vcpu->arch.time_page = NULL;
  926. }
  927. vcpu->arch.time = data;
  928. /* we verify if the enable bit is set... */
  929. if (!(data & 1))
  930. break;
  931. /* ...but clean it before doing the actual write */
  932. vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);
  933. vcpu->arch.time_page =
  934. gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);
  935. if (is_error_page(vcpu->arch.time_page)) {
  936. kvm_release_page_clean(vcpu->arch.time_page);
  937. vcpu->arch.time_page = NULL;
  938. }
  939. kvm_request_guest_time_update(vcpu);
  940. break;
  941. }
  942. case MSR_IA32_MCG_CTL:
  943. case MSR_IA32_MCG_STATUS:
  944. case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
  945. return set_msr_mce(vcpu, msr, data);
  946. /* Performance counters are not protected by a CPUID bit,
  947. * so we should check all of them in the generic path for the sake of
  948. * cross vendor migration.
  949. * Writing a zero into the event select MSRs disables them,
  950. * which we perfectly emulate ;-). Any other value should be at least
  951. * reported, some guests depend on them.
  952. */
  953. case MSR_P6_EVNTSEL0:
  954. case MSR_P6_EVNTSEL1:
  955. case MSR_K7_EVNTSEL0:
  956. case MSR_K7_EVNTSEL1:
  957. case MSR_K7_EVNTSEL2:
  958. case MSR_K7_EVNTSEL3:
  959. if (data != 0)
  960. pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
  961. "0x%x data 0x%llx\n", msr, data);
  962. break;
  963. /* at least RHEL 4 unconditionally writes to the perfctr registers,
  964. * so we ignore writes to make it happy.
  965. */
  966. case MSR_P6_PERFCTR0:
  967. case MSR_P6_PERFCTR1:
  968. case MSR_K7_PERFCTR0:
  969. case MSR_K7_PERFCTR1:
  970. case MSR_K7_PERFCTR2:
  971. case MSR_K7_PERFCTR3:
  972. pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
  973. "0x%x data 0x%llx\n", msr, data);
  974. break;
  975. default:
  976. if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
  977. return xen_hvm_config(vcpu, data);
  978. if (!ignore_msrs) {
  979. pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
  980. msr, data);
  981. return 1;
  982. } else {
  983. pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
  984. msr, data);
  985. break;
  986. }
  987. }
  988. return 0;
  989. }
  990. EXPORT_SYMBOL_GPL(kvm_set_msr_common);
  991. /*
  992. * Reads an msr value (of 'msr_index') into 'pdata'.
  993. * Returns 0 on success, non-0 otherwise.
  994. * Assumes vcpu_load() was already called.
  995. */
  996. int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
  997. {
  998. return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
  999. }
  1000. static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1001. {
  1002. u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
  1003. if (!msr_mtrr_valid(msr))
  1004. return 1;
  1005. if (msr == MSR_MTRRdefType)
  1006. *pdata = vcpu->arch.mtrr_state.def_type +
  1007. (vcpu->arch.mtrr_state.enabled << 10);
  1008. else if (msr == MSR_MTRRfix64K_00000)
  1009. *pdata = p[0];
  1010. else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
  1011. *pdata = p[1 + msr - MSR_MTRRfix16K_80000];
  1012. else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
  1013. *pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
  1014. else if (msr == MSR_IA32_CR_PAT)
  1015. *pdata = vcpu->arch.pat;
  1016. else { /* Variable MTRRs */
  1017. int idx, is_mtrr_mask;
  1018. u64 *pt;
  1019. idx = (msr - 0x200) / 2;
  1020. is_mtrr_mask = msr - 0x200 - 2 * idx;
  1021. if (!is_mtrr_mask)
  1022. pt =
  1023. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
  1024. else
  1025. pt =
  1026. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
  1027. *pdata = *pt;
  1028. }
  1029. return 0;
  1030. }
  1031. static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1032. {
  1033. u64 data;
  1034. u64 mcg_cap = vcpu->arch.mcg_cap;
  1035. unsigned bank_num = mcg_cap & 0xff;
  1036. switch (msr) {
  1037. case MSR_IA32_P5_MC_ADDR:
  1038. case MSR_IA32_P5_MC_TYPE:
  1039. data = 0;
  1040. break;
  1041. case MSR_IA32_MCG_CAP:
  1042. data = vcpu->arch.mcg_cap;
  1043. break;
  1044. case MSR_IA32_MCG_CTL:
  1045. if (!(mcg_cap & MCG_CTL_P))
  1046. return 1;
  1047. data = vcpu->arch.mcg_ctl;
  1048. break;
  1049. case MSR_IA32_MCG_STATUS:
  1050. data = vcpu->arch.mcg_status;
  1051. break;
  1052. default:
  1053. if (msr >= MSR_IA32_MC0_CTL &&
  1054. msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
  1055. u32 offset = msr - MSR_IA32_MC0_CTL;
  1056. data = vcpu->arch.mce_banks[offset];
  1057. break;
  1058. }
  1059. return 1;
  1060. }
  1061. *pdata = data;
  1062. return 0;
  1063. }
  1064. int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1065. {
  1066. u64 data;
  1067. switch (msr) {
  1068. case MSR_IA32_PLATFORM_ID:
  1069. case MSR_IA32_UCODE_REV:
  1070. case MSR_IA32_EBL_CR_POWERON:
  1071. case MSR_IA32_DEBUGCTLMSR:
  1072. case MSR_IA32_LASTBRANCHFROMIP:
  1073. case MSR_IA32_LASTBRANCHTOIP:
  1074. case MSR_IA32_LASTINTFROMIP:
  1075. case MSR_IA32_LASTINTTOIP:
  1076. case MSR_K8_SYSCFG:
  1077. case MSR_K7_HWCR:
  1078. case MSR_VM_HSAVE_PA:
  1079. case MSR_P6_PERFCTR0:
  1080. case MSR_P6_PERFCTR1:
  1081. case MSR_P6_EVNTSEL0:
  1082. case MSR_P6_EVNTSEL1:
  1083. case MSR_K7_EVNTSEL0:
  1084. case MSR_K7_PERFCTR0:
  1085. case MSR_K8_INT_PENDING_MSG:
  1086. case MSR_AMD64_NB_CFG:
  1087. case MSR_FAM10H_MMIO_CONF_BASE:
  1088. data = 0;
  1089. break;
  1090. case MSR_MTRRcap:
  1091. data = 0x500 | KVM_NR_VAR_MTRR;
  1092. break;
  1093. case 0x200 ... 0x2ff:
  1094. return get_msr_mtrr(vcpu, msr, pdata);
  1095. case 0xcd: /* fsb frequency */
  1096. data = 3;
  1097. break;
  1098. case MSR_IA32_APICBASE:
  1099. data = kvm_get_apic_base(vcpu);
  1100. break;
  1101. case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
  1102. return kvm_x2apic_msr_read(vcpu, msr, pdata);
  1103. break;
  1104. case MSR_IA32_MISC_ENABLE:
  1105. data = vcpu->arch.ia32_misc_enable_msr;
  1106. break;
  1107. case MSR_IA32_PERF_STATUS:
  1108. /* TSC increment by tick */
  1109. data = 1000ULL;
  1110. /* CPU multiplier */
  1111. data |= (((uint64_t)4ULL) << 40);
  1112. break;
  1113. case MSR_EFER:
  1114. data = vcpu->arch.shadow_efer;
  1115. break;
  1116. case MSR_KVM_WALL_CLOCK:
  1117. data = vcpu->kvm->arch.wall_clock;
  1118. break;
  1119. case MSR_KVM_SYSTEM_TIME:
  1120. data = vcpu->arch.time;
  1121. break;
  1122. case MSR_IA32_P5_MC_ADDR:
  1123. case MSR_IA32_P5_MC_TYPE:
  1124. case MSR_IA32_MCG_CAP:
  1125. case MSR_IA32_MCG_CTL:
  1126. case MSR_IA32_MCG_STATUS:
  1127. case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
  1128. return get_msr_mce(vcpu, msr, pdata);
  1129. default:
  1130. if (!ignore_msrs) {
  1131. pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
  1132. return 1;
  1133. } else {
  1134. pr_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
  1135. data = 0;
  1136. }
  1137. break;
  1138. }
  1139. *pdata = data;
  1140. return 0;
  1141. }
  1142. EXPORT_SYMBOL_GPL(kvm_get_msr_common);
  1143. /*
  1144. * Read or write a bunch of msrs. All parameters are kernel addresses.
  1145. *
  1146. * @return number of msrs set successfully.
  1147. */
  1148. static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
  1149. struct kvm_msr_entry *entries,
  1150. int (*do_msr)(struct kvm_vcpu *vcpu,
  1151. unsigned index, u64 *data))
  1152. {
  1153. int i, idx;
  1154. vcpu_load(vcpu);
  1155. idx = srcu_read_lock(&vcpu->kvm->srcu);
  1156. for (i = 0; i < msrs->nmsrs; ++i)
  1157. if (do_msr(vcpu, entries[i].index, &entries[i].data))
  1158. break;
  1159. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  1160. vcpu_put(vcpu);
  1161. return i;
  1162. }
  1163. /*
  1164. * Read or write a bunch of msrs. Parameters are user addresses.
  1165. *
  1166. * @return number of msrs set successfully.
  1167. */
  1168. static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
  1169. int (*do_msr)(struct kvm_vcpu *vcpu,
  1170. unsigned index, u64 *data),
  1171. int writeback)
  1172. {
  1173. struct kvm_msrs msrs;
  1174. struct kvm_msr_entry *entries;
  1175. int r, n;
  1176. unsigned size;
  1177. r = -EFAULT;
  1178. if (copy_from_user(&msrs, user_msrs, sizeof msrs))
  1179. goto out;
  1180. r = -E2BIG;
  1181. if (msrs.nmsrs >= MAX_IO_MSRS)
  1182. goto out;
  1183. r = -ENOMEM;
  1184. size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
  1185. entries = vmalloc(size);
  1186. if (!entries)
  1187. goto out;
  1188. r = -EFAULT;
  1189. if (copy_from_user(entries, user_msrs->entries, size))
  1190. goto out_free;
  1191. r = n = __msr_io(vcpu, &msrs, entries, do_msr);
  1192. if (r < 0)
  1193. goto out_free;
  1194. r = -EFAULT;
  1195. if (writeback && copy_to_user(user_msrs->entries, entries, size))
  1196. goto out_free;
  1197. r = n;
  1198. out_free:
  1199. vfree(entries);
  1200. out:
  1201. return r;
  1202. }
  1203. int kvm_dev_ioctl_check_extension(long ext)
  1204. {
  1205. int r;
  1206. switch (ext) {
  1207. case KVM_CAP_IRQCHIP:
  1208. case KVM_CAP_HLT:
  1209. case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
  1210. case KVM_CAP_SET_TSS_ADDR:
  1211. case KVM_CAP_EXT_CPUID:
  1212. case KVM_CAP_CLOCKSOURCE:
  1213. case KVM_CAP_PIT:
  1214. case KVM_CAP_NOP_IO_DELAY:
  1215. case KVM_CAP_MP_STATE:
  1216. case KVM_CAP_SYNC_MMU:
  1217. case KVM_CAP_REINJECT_CONTROL:
  1218. case KVM_CAP_IRQ_INJECT_STATUS:
  1219. case KVM_CAP_ASSIGN_DEV_IRQ:
  1220. case KVM_CAP_IRQFD:
  1221. case KVM_CAP_IOEVENTFD:
  1222. case KVM_CAP_PIT2:
  1223. case KVM_CAP_PIT_STATE2:
  1224. case KVM_CAP_SET_IDENTITY_MAP_ADDR:
  1225. case KVM_CAP_XEN_HVM:
  1226. case KVM_CAP_ADJUST_CLOCK:
  1227. case KVM_CAP_VCPU_EVENTS:
  1228. r = 1;
  1229. break;
  1230. case KVM_CAP_COALESCED_MMIO:
  1231. r = KVM_COALESCED_MMIO_PAGE_OFFSET;
  1232. break;
  1233. case KVM_CAP_VAPIC:
  1234. r = !kvm_x86_ops->cpu_has_accelerated_tpr();
  1235. break;
  1236. case KVM_CAP_NR_VCPUS:
  1237. r = KVM_MAX_VCPUS;
  1238. break;
  1239. case KVM_CAP_NR_MEMSLOTS:
  1240. r = KVM_MEMORY_SLOTS;
  1241. break;
  1242. case KVM_CAP_PV_MMU: /* obsolete */
  1243. r = 0;
  1244. break;
  1245. case KVM_CAP_IOMMU:
  1246. r = iommu_found();
  1247. break;
  1248. case KVM_CAP_MCE:
  1249. r = KVM_MAX_MCE_BANKS;
  1250. break;
  1251. default:
  1252. r = 0;
  1253. break;
  1254. }
  1255. return r;
  1256. }
  1257. long kvm_arch_dev_ioctl(struct file *filp,
  1258. unsigned int ioctl, unsigned long arg)
  1259. {
  1260. void __user *argp = (void __user *)arg;
  1261. long r;
  1262. switch (ioctl) {
  1263. case KVM_GET_MSR_INDEX_LIST: {
  1264. struct kvm_msr_list __user *user_msr_list = argp;
  1265. struct kvm_msr_list msr_list;
  1266. unsigned n;
  1267. r = -EFAULT;
  1268. if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
  1269. goto out;
  1270. n = msr_list.nmsrs;
  1271. msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
  1272. if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
  1273. goto out;
  1274. r = -E2BIG;
  1275. if (n < msr_list.nmsrs)
  1276. goto out;
  1277. r = -EFAULT;
  1278. if (copy_to_user(user_msr_list->indices, &msrs_to_save,
  1279. num_msrs_to_save * sizeof(u32)))
  1280. goto out;
  1281. if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
  1282. &emulated_msrs,
  1283. ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
  1284. goto out;
  1285. r = 0;
  1286. break;
  1287. }
  1288. case KVM_GET_SUPPORTED_CPUID: {
  1289. struct kvm_cpuid2 __user *cpuid_arg = argp;
  1290. struct kvm_cpuid2 cpuid;
  1291. r = -EFAULT;
  1292. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1293. goto out;
  1294. r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
  1295. cpuid_arg->entries);
  1296. if (r)
  1297. goto out;
  1298. r = -EFAULT;
  1299. if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
  1300. goto out;
  1301. r = 0;
  1302. break;
  1303. }
  1304. case KVM_X86_GET_MCE_CAP_SUPPORTED: {
  1305. u64 mce_cap;
  1306. mce_cap = KVM_MCE_CAP_SUPPORTED;
  1307. r = -EFAULT;
  1308. if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
  1309. goto out;
  1310. r = 0;
  1311. break;
  1312. }
  1313. default:
  1314. r = -EINVAL;
  1315. }
  1316. out:
  1317. return r;
  1318. }
  1319. void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  1320. {
  1321. kvm_x86_ops->vcpu_load(vcpu, cpu);
  1322. if (unlikely(per_cpu(cpu_tsc_khz, cpu) == 0)) {
  1323. unsigned long khz = cpufreq_quick_get(cpu);
  1324. if (!khz)
  1325. khz = tsc_khz;
  1326. per_cpu(cpu_tsc_khz, cpu) = khz;
  1327. }
  1328. kvm_request_guest_time_update(vcpu);
  1329. }
  1330. void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
  1331. {
  1332. kvm_x86_ops->vcpu_put(vcpu);
  1333. kvm_put_guest_fpu(vcpu);
  1334. }
  1335. static int is_efer_nx(void)
  1336. {
  1337. unsigned long long efer = 0;
  1338. rdmsrl_safe(MSR_EFER, &efer);
  1339. return efer & EFER_NX;
  1340. }
  1341. static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
  1342. {
  1343. int i;
  1344. struct kvm_cpuid_entry2 *e, *entry;
  1345. entry = NULL;
  1346. for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
  1347. e = &vcpu->arch.cpuid_entries[i];
  1348. if (e->function == 0x80000001) {
  1349. entry = e;
  1350. break;
  1351. }
  1352. }
  1353. if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) {
  1354. entry->edx &= ~(1 << 20);
  1355. printk(KERN_INFO "kvm: guest NX capability removed\n");
  1356. }
  1357. }
  1358. /* when an old userspace process fills a new kernel module */
  1359. static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
  1360. struct kvm_cpuid *cpuid,
  1361. struct kvm_cpuid_entry __user *entries)
  1362. {
  1363. int r, i;
  1364. struct kvm_cpuid_entry *cpuid_entries;
  1365. r = -E2BIG;
  1366. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  1367. goto out;
  1368. r = -ENOMEM;
  1369. cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent);
  1370. if (!cpuid_entries)
  1371. goto out;
  1372. r = -EFAULT;
  1373. if (copy_from_user(cpuid_entries, entries,
  1374. cpuid->nent * sizeof(struct kvm_cpuid_entry)))
  1375. goto out_free;
  1376. for (i = 0; i < cpuid->nent; i++) {
  1377. vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
  1378. vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
  1379. vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
  1380. vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
  1381. vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
  1382. vcpu->arch.cpuid_entries[i].index = 0;
  1383. vcpu->arch.cpuid_entries[i].flags = 0;
  1384. vcpu->arch.cpuid_entries[i].padding[0] = 0;
  1385. vcpu->arch.cpuid_entries[i].padding[1] = 0;
  1386. vcpu->arch.cpuid_entries[i].padding[2] = 0;
  1387. }
  1388. vcpu->arch.cpuid_nent = cpuid->nent;
  1389. cpuid_fix_nx_cap(vcpu);
  1390. r = 0;
  1391. kvm_apic_set_version(vcpu);
  1392. kvm_x86_ops->cpuid_update(vcpu);
  1393. out_free:
  1394. vfree(cpuid_entries);
  1395. out:
  1396. return r;
  1397. }
  1398. static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
  1399. struct kvm_cpuid2 *cpuid,
  1400. struct kvm_cpuid_entry2 __user *entries)
  1401. {
  1402. int r;
  1403. r = -E2BIG;
  1404. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  1405. goto out;
  1406. r = -EFAULT;
  1407. if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
  1408. cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
  1409. goto out;
  1410. vcpu->arch.cpuid_nent = cpuid->nent;
  1411. kvm_apic_set_version(vcpu);
  1412. kvm_x86_ops->cpuid_update(vcpu);
  1413. return 0;
  1414. out:
  1415. return r;
  1416. }
  1417. static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
  1418. struct kvm_cpuid2 *cpuid,
  1419. struct kvm_cpuid_entry2 __user *entries)
  1420. {
  1421. int r;
  1422. r = -E2BIG;
  1423. if (cpuid->nent < vcpu->arch.cpuid_nent)
  1424. goto out;
  1425. r = -EFAULT;
  1426. if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
  1427. vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
  1428. goto out;
  1429. return 0;
  1430. out:
  1431. cpuid->nent = vcpu->arch.cpuid_nent;
  1432. return r;
  1433. }
  1434. static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
  1435. u32 index)
  1436. {
  1437. entry->function = function;
  1438. entry->index = index;
  1439. cpuid_count(entry->function, entry->index,
  1440. &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
  1441. entry->flags = 0;
  1442. }
  1443. #define F(x) bit(X86_FEATURE_##x)
  1444. static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
  1445. u32 index, int *nent, int maxnent)
  1446. {
  1447. unsigned f_nx = is_efer_nx() ? F(NX) : 0;
  1448. #ifdef CONFIG_X86_64
  1449. unsigned f_gbpages = (kvm_x86_ops->get_lpage_level() == PT_PDPE_LEVEL)
  1450. ? F(GBPAGES) : 0;
  1451. unsigned f_lm = F(LM);
  1452. #else
  1453. unsigned f_gbpages = 0;
  1454. unsigned f_lm = 0;
  1455. #endif
  1456. unsigned f_rdtscp = kvm_x86_ops->rdtscp_supported() ? F(RDTSCP) : 0;
  1457. /* cpuid 1.edx */
  1458. const u32 kvm_supported_word0_x86_features =
  1459. F(FPU) | F(VME) | F(DE) | F(PSE) |
  1460. F(TSC) | F(MSR) | F(PAE) | F(MCE) |
  1461. F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
  1462. F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
  1463. F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLSH) |
  1464. 0 /* Reserved, DS, ACPI */ | F(MMX) |
  1465. F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
  1466. 0 /* HTT, TM, Reserved, PBE */;
  1467. /* cpuid 0x80000001.edx */
  1468. const u32 kvm_supported_word1_x86_features =
  1469. F(FPU) | F(VME) | F(DE) | F(PSE) |
  1470. F(TSC) | F(MSR) | F(PAE) | F(MCE) |
  1471. F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
  1472. F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
  1473. F(PAT) | F(PSE36) | 0 /* Reserved */ |
  1474. f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
  1475. F(FXSR) | F(FXSR_OPT) | f_gbpages | f_rdtscp |
  1476. 0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW);
  1477. /* cpuid 1.ecx */
  1478. const u32 kvm_supported_word4_x86_features =
  1479. F(XMM3) | 0 /* Reserved, DTES64, MONITOR */ |
  1480. 0 /* DS-CPL, VMX, SMX, EST */ |
  1481. 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
  1482. 0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ |
  1483. 0 /* Reserved, DCA */ | F(XMM4_1) |
  1484. F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
  1485. 0 /* Reserved, XSAVE, OSXSAVE */;
  1486. /* cpuid 0x80000001.ecx */
  1487. const u32 kvm_supported_word6_x86_features =
  1488. F(LAHF_LM) | F(CMP_LEGACY) | F(SVM) | 0 /* ExtApicSpace */ |
  1489. F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
  1490. F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(SSE5) |
  1491. 0 /* SKINIT */ | 0 /* WDT */;
  1492. /* all calls to cpuid_count() should be made on the same cpu */
  1493. get_cpu();
  1494. do_cpuid_1_ent(entry, function, index);
  1495. ++*nent;
  1496. switch (function) {
  1497. case 0:
  1498. entry->eax = min(entry->eax, (u32)0xb);
  1499. break;
  1500. case 1:
  1501. entry->edx &= kvm_supported_word0_x86_features;
  1502. entry->ecx &= kvm_supported_word4_x86_features;
  1503. /* we support x2apic emulation even if host does not support
  1504. * it since we emulate x2apic in software */
  1505. entry->ecx |= F(X2APIC);
  1506. break;
  1507. /* function 2 entries are STATEFUL. That is, repeated cpuid commands
  1508. * may return different values. This forces us to get_cpu() before
  1509. * issuing the first command, and also to emulate this annoying behavior
  1510. * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
  1511. case 2: {
  1512. int t, times = entry->eax & 0xff;
  1513. entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
  1514. entry->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
  1515. for (t = 1; t < times && *nent < maxnent; ++t) {
  1516. do_cpuid_1_ent(&entry[t], function, 0);
  1517. entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
  1518. ++*nent;
  1519. }
  1520. break;
  1521. }
  1522. /* function 4 and 0xb have additional index. */
  1523. case 4: {
  1524. int i, cache_type;
  1525. entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1526. /* read more entries until cache_type is zero */
  1527. for (i = 1; *nent < maxnent; ++i) {
  1528. cache_type = entry[i - 1].eax & 0x1f;
  1529. if (!cache_type)
  1530. break;
  1531. do_cpuid_1_ent(&entry[i], function, i);
  1532. entry[i].flags |=
  1533. KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1534. ++*nent;
  1535. }
  1536. break;
  1537. }
  1538. case 0xb: {
  1539. int i, level_type;
  1540. entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1541. /* read more entries until level_type is zero */
  1542. for (i = 1; *nent < maxnent; ++i) {
  1543. level_type = entry[i - 1].ecx & 0xff00;
  1544. if (!level_type)
  1545. break;
  1546. do_cpuid_1_ent(&entry[i], function, i);
  1547. entry[i].flags |=
  1548. KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1549. ++*nent;
  1550. }
  1551. break;
  1552. }
  1553. case 0x80000000:
  1554. entry->eax = min(entry->eax, 0x8000001a);
  1555. break;
  1556. case 0x80000001:
  1557. entry->edx &= kvm_supported_word1_x86_features;
  1558. entry->ecx &= kvm_supported_word6_x86_features;
  1559. break;
  1560. }
  1561. put_cpu();
  1562. }
  1563. #undef F
  1564. static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  1565. struct kvm_cpuid_entry2 __user *entries)
  1566. {
  1567. struct kvm_cpuid_entry2 *cpuid_entries;
  1568. int limit, nent = 0, r = -E2BIG;
  1569. u32 func;
  1570. if (cpuid->nent < 1)
  1571. goto out;
  1572. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  1573. cpuid->nent = KVM_MAX_CPUID_ENTRIES;
  1574. r = -ENOMEM;
  1575. cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent);
  1576. if (!cpuid_entries)
  1577. goto out;
  1578. do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent);
  1579. limit = cpuid_entries[0].eax;
  1580. for (func = 1; func <= limit && nent < cpuid->nent; ++func)
  1581. do_cpuid_ent(&cpuid_entries[nent], func, 0,
  1582. &nent, cpuid->nent);
  1583. r = -E2BIG;
  1584. if (nent >= cpuid->nent)
  1585. goto out_free;
  1586. do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent);
  1587. limit = cpuid_entries[nent - 1].eax;
  1588. for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func)
  1589. do_cpuid_ent(&cpuid_entries[nent], func, 0,
  1590. &nent, cpuid->nent);
  1591. r = -E2BIG;
  1592. if (nent >= cpuid->nent)
  1593. goto out_free;
  1594. r = -EFAULT;
  1595. if (copy_to_user(entries, cpuid_entries,
  1596. nent * sizeof(struct kvm_cpuid_entry2)))
  1597. goto out_free;
  1598. cpuid->nent = nent;
  1599. r = 0;
  1600. out_free:
  1601. vfree(cpuid_entries);
  1602. out:
  1603. return r;
  1604. }
  1605. static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
  1606. struct kvm_lapic_state *s)
  1607. {
  1608. vcpu_load(vcpu);
  1609. memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
  1610. vcpu_put(vcpu);
  1611. return 0;
  1612. }
  1613. static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
  1614. struct kvm_lapic_state *s)
  1615. {
  1616. vcpu_load(vcpu);
  1617. memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
  1618. kvm_apic_post_state_restore(vcpu);
  1619. update_cr8_intercept(vcpu);
  1620. vcpu_put(vcpu);
  1621. return 0;
  1622. }
  1623. static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
  1624. struct kvm_interrupt *irq)
  1625. {
  1626. if (irq->irq < 0 || irq->irq >= 256)
  1627. return -EINVAL;
  1628. if (irqchip_in_kernel(vcpu->kvm))
  1629. return -ENXIO;
  1630. vcpu_load(vcpu);
  1631. kvm_queue_interrupt(vcpu, irq->irq, false);
  1632. vcpu_put(vcpu);
  1633. return 0;
  1634. }
  1635. static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
  1636. {
  1637. vcpu_load(vcpu);
  1638. kvm_inject_nmi(vcpu);
  1639. vcpu_put(vcpu);
  1640. return 0;
  1641. }
  1642. static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
  1643. struct kvm_tpr_access_ctl *tac)
  1644. {
  1645. if (tac->flags)
  1646. return -EINVAL;
  1647. vcpu->arch.tpr_access_reporting = !!tac->enabled;
  1648. return 0;
  1649. }
  1650. static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
  1651. u64 mcg_cap)
  1652. {
  1653. int r;
  1654. unsigned bank_num = mcg_cap & 0xff, bank;
  1655. r = -EINVAL;
  1656. if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
  1657. goto out;
  1658. if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
  1659. goto out;
  1660. r = 0;
  1661. vcpu->arch.mcg_cap = mcg_cap;
  1662. /* Init IA32_MCG_CTL to all 1s */
  1663. if (mcg_cap & MCG_CTL_P)
  1664. vcpu->arch.mcg_ctl = ~(u64)0;
  1665. /* Init IA32_MCi_CTL to all 1s */
  1666. for (bank = 0; bank < bank_num; bank++)
  1667. vcpu->arch.mce_banks[bank*4] = ~(u64)0;
  1668. out:
  1669. return r;
  1670. }
  1671. static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
  1672. struct kvm_x86_mce *mce)
  1673. {
  1674. u64 mcg_cap = vcpu->arch.mcg_cap;
  1675. unsigned bank_num = mcg_cap & 0xff;
  1676. u64 *banks = vcpu->arch.mce_banks;
  1677. if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
  1678. return -EINVAL;
  1679. /*
  1680. * if IA32_MCG_CTL is not all 1s, the uncorrected error
  1681. * reporting is disabled
  1682. */
  1683. if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
  1684. vcpu->arch.mcg_ctl != ~(u64)0)
  1685. return 0;
  1686. banks += 4 * mce->bank;
  1687. /*
  1688. * if IA32_MCi_CTL is not all 1s, the uncorrected error
  1689. * reporting is disabled for the bank
  1690. */
  1691. if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
  1692. return 0;
  1693. if (mce->status & MCI_STATUS_UC) {
  1694. if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
  1695. !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
  1696. printk(KERN_DEBUG "kvm: set_mce: "
  1697. "injects mce exception while "
  1698. "previous one is in progress!\n");
  1699. set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
  1700. return 0;
  1701. }
  1702. if (banks[1] & MCI_STATUS_VAL)
  1703. mce->status |= MCI_STATUS_OVER;
  1704. banks[2] = mce->addr;
  1705. banks[3] = mce->misc;
  1706. vcpu->arch.mcg_status = mce->mcg_status;
  1707. banks[1] = mce->status;
  1708. kvm_queue_exception(vcpu, MC_VECTOR);
  1709. } else if (!(banks[1] & MCI_STATUS_VAL)
  1710. || !(banks[1] & MCI_STATUS_UC)) {
  1711. if (banks[1] & MCI_STATUS_VAL)
  1712. mce->status |= MCI_STATUS_OVER;
  1713. banks[2] = mce->addr;
  1714. banks[3] = mce->misc;
  1715. banks[1] = mce->status;
  1716. } else
  1717. banks[1] |= MCI_STATUS_OVER;
  1718. return 0;
  1719. }
  1720. static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
  1721. struct kvm_vcpu_events *events)
  1722. {
  1723. vcpu_load(vcpu);
  1724. events->exception.injected = vcpu->arch.exception.pending;
  1725. events->exception.nr = vcpu->arch.exception.nr;
  1726. events->exception.has_error_code = vcpu->arch.exception.has_error_code;
  1727. events->exception.error_code = vcpu->arch.exception.error_code;
  1728. events->interrupt.injected = vcpu->arch.interrupt.pending;
  1729. events->interrupt.nr = vcpu->arch.interrupt.nr;
  1730. events->interrupt.soft = vcpu->arch.interrupt.soft;
  1731. events->nmi.injected = vcpu->arch.nmi_injected;
  1732. events->nmi.pending = vcpu->arch.nmi_pending;
  1733. events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
  1734. events->sipi_vector = vcpu->arch.sipi_vector;
  1735. events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
  1736. | KVM_VCPUEVENT_VALID_SIPI_VECTOR);
  1737. vcpu_put(vcpu);
  1738. }
  1739. static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
  1740. struct kvm_vcpu_events *events)
  1741. {
  1742. if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
  1743. | KVM_VCPUEVENT_VALID_SIPI_VECTOR))
  1744. return -EINVAL;
  1745. vcpu_load(vcpu);
  1746. vcpu->arch.exception.pending = events->exception.injected;
  1747. vcpu->arch.exception.nr = events->exception.nr;
  1748. vcpu->arch.exception.has_error_code = events->exception.has_error_code;
  1749. vcpu->arch.exception.error_code = events->exception.error_code;
  1750. vcpu->arch.interrupt.pending = events->interrupt.injected;
  1751. vcpu->arch.interrupt.nr = events->interrupt.nr;
  1752. vcpu->arch.interrupt.soft = events->interrupt.soft;
  1753. if (vcpu->arch.interrupt.pending && irqchip_in_kernel(vcpu->kvm))
  1754. kvm_pic_clear_isr_ack(vcpu->kvm);
  1755. vcpu->arch.nmi_injected = events->nmi.injected;
  1756. if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
  1757. vcpu->arch.nmi_pending = events->nmi.pending;
  1758. kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);
  1759. if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR)
  1760. vcpu->arch.sipi_vector = events->sipi_vector;
  1761. vcpu_put(vcpu);
  1762. return 0;
  1763. }
  1764. long kvm_arch_vcpu_ioctl(struct file *filp,
  1765. unsigned int ioctl, unsigned long arg)
  1766. {
  1767. struct kvm_vcpu *vcpu = filp->private_data;
  1768. void __user *argp = (void __user *)arg;
  1769. int r;
  1770. struct kvm_lapic_state *lapic = NULL;
  1771. switch (ioctl) {
  1772. case KVM_GET_LAPIC: {
  1773. r = -EINVAL;
  1774. if (!vcpu->arch.apic)
  1775. goto out;
  1776. lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
  1777. r = -ENOMEM;
  1778. if (!lapic)
  1779. goto out;
  1780. r = kvm_vcpu_ioctl_get_lapic(vcpu, lapic);
  1781. if (r)
  1782. goto out;
  1783. r = -EFAULT;
  1784. if (copy_to_user(argp, lapic, sizeof(struct kvm_lapic_state)))
  1785. goto out;
  1786. r = 0;
  1787. break;
  1788. }
  1789. case KVM_SET_LAPIC: {
  1790. r = -EINVAL;
  1791. if (!vcpu->arch.apic)
  1792. goto out;
  1793. lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
  1794. r = -ENOMEM;
  1795. if (!lapic)
  1796. goto out;
  1797. r = -EFAULT;
  1798. if (copy_from_user(lapic, argp, sizeof(struct kvm_lapic_state)))
  1799. goto out;
  1800. r = kvm_vcpu_ioctl_set_lapic(vcpu, lapic);
  1801. if (r)
  1802. goto out;
  1803. r = 0;
  1804. break;
  1805. }
  1806. case KVM_INTERRUPT: {
  1807. struct kvm_interrupt irq;
  1808. r = -EFAULT;
  1809. if (copy_from_user(&irq, argp, sizeof irq))
  1810. goto out;
  1811. r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
  1812. if (r)
  1813. goto out;
  1814. r = 0;
  1815. break;
  1816. }
  1817. case KVM_NMI: {
  1818. r = kvm_vcpu_ioctl_nmi(vcpu);
  1819. if (r)
  1820. goto out;
  1821. r = 0;
  1822. break;
  1823. }
  1824. case KVM_SET_CPUID: {
  1825. struct kvm_cpuid __user *cpuid_arg = argp;
  1826. struct kvm_cpuid cpuid;
  1827. r = -EFAULT;
  1828. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1829. goto out;
  1830. r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
  1831. if (r)
  1832. goto out;
  1833. break;
  1834. }
  1835. case KVM_SET_CPUID2: {
  1836. struct kvm_cpuid2 __user *cpuid_arg = argp;
  1837. struct kvm_cpuid2 cpuid;
  1838. r = -EFAULT;
  1839. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1840. goto out;
  1841. r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
  1842. cpuid_arg->entries);
  1843. if (r)
  1844. goto out;
  1845. break;
  1846. }
  1847. case KVM_GET_CPUID2: {
  1848. struct kvm_cpuid2 __user *cpuid_arg = argp;
  1849. struct kvm_cpuid2 cpuid;
  1850. r = -EFAULT;
  1851. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1852. goto out;
  1853. r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
  1854. cpuid_arg->entries);
  1855. if (r)
  1856. goto out;
  1857. r = -EFAULT;
  1858. if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
  1859. goto out;
  1860. r = 0;
  1861. break;
  1862. }
  1863. case KVM_GET_MSRS:
  1864. r = msr_io(vcpu, argp, kvm_get_msr, 1);
  1865. break;
  1866. case KVM_SET_MSRS:
  1867. r = msr_io(vcpu, argp, do_set_msr, 0);
  1868. break;
  1869. case KVM_TPR_ACCESS_REPORTING: {
  1870. struct kvm_tpr_access_ctl tac;
  1871. r = -EFAULT;
  1872. if (copy_from_user(&tac, argp, sizeof tac))
  1873. goto out;
  1874. r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
  1875. if (r)
  1876. goto out;
  1877. r = -EFAULT;
  1878. if (copy_to_user(argp, &tac, sizeof tac))
  1879. goto out;
  1880. r = 0;
  1881. break;
  1882. };
  1883. case KVM_SET_VAPIC_ADDR: {
  1884. struct kvm_vapic_addr va;
  1885. r = -EINVAL;
  1886. if (!irqchip_in_kernel(vcpu->kvm))
  1887. goto out;
  1888. r = -EFAULT;
  1889. if (copy_from_user(&va, argp, sizeof va))
  1890. goto out;
  1891. r = 0;
  1892. kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
  1893. break;
  1894. }
  1895. case KVM_X86_SETUP_MCE: {
  1896. u64 mcg_cap;
  1897. r = -EFAULT;
  1898. if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
  1899. goto out;
  1900. r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
  1901. break;
  1902. }
  1903. case KVM_X86_SET_MCE: {
  1904. struct kvm_x86_mce mce;
  1905. r = -EFAULT;
  1906. if (copy_from_user(&mce, argp, sizeof mce))
  1907. goto out;
  1908. r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
  1909. break;
  1910. }
  1911. case KVM_GET_VCPU_EVENTS: {
  1912. struct kvm_vcpu_events events;
  1913. kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
  1914. r = -EFAULT;
  1915. if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
  1916. break;
  1917. r = 0;
  1918. break;
  1919. }
  1920. case KVM_SET_VCPU_EVENTS: {
  1921. struct kvm_vcpu_events events;
  1922. r = -EFAULT;
  1923. if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
  1924. break;
  1925. r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
  1926. break;
  1927. }
  1928. default:
  1929. r = -EINVAL;
  1930. }
  1931. out:
  1932. kfree(lapic);
  1933. return r;
  1934. }
  1935. static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
  1936. {
  1937. int ret;
  1938. if (addr > (unsigned int)(-3 * PAGE_SIZE))
  1939. return -1;
  1940. ret = kvm_x86_ops->set_tss_addr(kvm, addr);
  1941. return ret;
  1942. }
  1943. static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
  1944. u64 ident_addr)
  1945. {
  1946. kvm->arch.ept_identity_map_addr = ident_addr;
  1947. return 0;
  1948. }
  1949. static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
  1950. u32 kvm_nr_mmu_pages)
  1951. {
  1952. if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
  1953. return -EINVAL;
  1954. mutex_lock(&kvm->slots_lock);
  1955. spin_lock(&kvm->mmu_lock);
  1956. kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
  1957. kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
  1958. spin_unlock(&kvm->mmu_lock);
  1959. mutex_unlock(&kvm->slots_lock);
  1960. return 0;
  1961. }
  1962. static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
  1963. {
  1964. return kvm->arch.n_alloc_mmu_pages;
  1965. }
  1966. gfn_t unalias_gfn_instantiation(struct kvm *kvm, gfn_t gfn)
  1967. {
  1968. int i;
  1969. struct kvm_mem_alias *alias;
  1970. struct kvm_mem_aliases *aliases;
  1971. aliases = rcu_dereference(kvm->arch.aliases);
  1972. for (i = 0; i < aliases->naliases; ++i) {
  1973. alias = &aliases->aliases[i];
  1974. if (alias->flags & KVM_ALIAS_INVALID)
  1975. continue;
  1976. if (gfn >= alias->base_gfn
  1977. && gfn < alias->base_gfn + alias->npages)
  1978. return alias->target_gfn + gfn - alias->base_gfn;
  1979. }
  1980. return gfn;
  1981. }
  1982. gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
  1983. {
  1984. int i;
  1985. struct kvm_mem_alias *alias;
  1986. struct kvm_mem_aliases *aliases;
  1987. aliases = rcu_dereference(kvm->arch.aliases);
  1988. for (i = 0; i < aliases->naliases; ++i) {
  1989. alias = &aliases->aliases[i];
  1990. if (gfn >= alias->base_gfn
  1991. && gfn < alias->base_gfn + alias->npages)
  1992. return alias->target_gfn + gfn - alias->base_gfn;
  1993. }
  1994. return gfn;
  1995. }
  1996. /*
  1997. * Set a new alias region. Aliases map a portion of physical memory into
  1998. * another portion. This is useful for memory windows, for example the PC
  1999. * VGA region.
  2000. */
  2001. static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
  2002. struct kvm_memory_alias *alias)
  2003. {
  2004. int r, n;
  2005. struct kvm_mem_alias *p;
  2006. struct kvm_mem_aliases *aliases, *old_aliases;
  2007. r = -EINVAL;
  2008. /* General sanity checks */
  2009. if (alias->memory_size & (PAGE_SIZE - 1))
  2010. goto out;
  2011. if (alias->guest_phys_addr & (PAGE_SIZE - 1))
  2012. goto out;
  2013. if (alias->slot >= KVM_ALIAS_SLOTS)
  2014. goto out;
  2015. if (alias->guest_phys_addr + alias->memory_size
  2016. < alias->guest_phys_addr)
  2017. goto out;
  2018. if (alias->target_phys_addr + alias->memory_size
  2019. < alias->target_phys_addr)
  2020. goto out;
  2021. r = -ENOMEM;
  2022. aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
  2023. if (!aliases)
  2024. goto out;
  2025. mutex_lock(&kvm->slots_lock);
  2026. /* invalidate any gfn reference in case of deletion/shrinking */
  2027. memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
  2028. aliases->aliases[alias->slot].flags |= KVM_ALIAS_INVALID;
  2029. old_aliases = kvm->arch.aliases;
  2030. rcu_assign_pointer(kvm->arch.aliases, aliases);
  2031. synchronize_srcu_expedited(&kvm->srcu);
  2032. kvm_mmu_zap_all(kvm);
  2033. kfree(old_aliases);
  2034. r = -ENOMEM;
  2035. aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
  2036. if (!aliases)
  2037. goto out_unlock;
  2038. memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
  2039. p = &aliases->aliases[alias->slot];
  2040. p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
  2041. p->npages = alias->memory_size >> PAGE_SHIFT;
  2042. p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
  2043. p->flags &= ~(KVM_ALIAS_INVALID);
  2044. for (n = KVM_ALIAS_SLOTS; n > 0; --n)
  2045. if (aliases->aliases[n - 1].npages)
  2046. break;
  2047. aliases->naliases = n;
  2048. old_aliases = kvm->arch.aliases;
  2049. rcu_assign_pointer(kvm->arch.aliases, aliases);
  2050. synchronize_srcu_expedited(&kvm->srcu);
  2051. kfree(old_aliases);
  2052. r = 0;
  2053. out_unlock:
  2054. mutex_unlock(&kvm->slots_lock);
  2055. out:
  2056. return r;
  2057. }
  2058. static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
  2059. {
  2060. int r;
  2061. r = 0;
  2062. switch (chip->chip_id) {
  2063. case KVM_IRQCHIP_PIC_MASTER:
  2064. memcpy(&chip->chip.pic,
  2065. &pic_irqchip(kvm)->pics[0],
  2066. sizeof(struct kvm_pic_state));
  2067. break;
  2068. case KVM_IRQCHIP_PIC_SLAVE:
  2069. memcpy(&chip->chip.pic,
  2070. &pic_irqchip(kvm)->pics[1],
  2071. sizeof(struct kvm_pic_state));
  2072. break;
  2073. case KVM_IRQCHIP_IOAPIC:
  2074. r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
  2075. break;
  2076. default:
  2077. r = -EINVAL;
  2078. break;
  2079. }
  2080. return r;
  2081. }
  2082. static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
  2083. {
  2084. int r;
  2085. r = 0;
  2086. switch (chip->chip_id) {
  2087. case KVM_IRQCHIP_PIC_MASTER:
  2088. spin_lock(&pic_irqchip(kvm)->lock);
  2089. memcpy(&pic_irqchip(kvm)->pics[0],
  2090. &chip->chip.pic,
  2091. sizeof(struct kvm_pic_state));
  2092. spin_unlock(&pic_irqchip(kvm)->lock);
  2093. break;
  2094. case KVM_IRQCHIP_PIC_SLAVE:
  2095. spin_lock(&pic_irqchip(kvm)->lock);
  2096. memcpy(&pic_irqchip(kvm)->pics[1],
  2097. &chip->chip.pic,
  2098. sizeof(struct kvm_pic_state));
  2099. spin_unlock(&pic_irqchip(kvm)->lock);
  2100. break;
  2101. case KVM_IRQCHIP_IOAPIC:
  2102. r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
  2103. break;
  2104. default:
  2105. r = -EINVAL;
  2106. break;
  2107. }
  2108. kvm_pic_update_irq(pic_irqchip(kvm));
  2109. return r;
  2110. }
  2111. static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
  2112. {
  2113. int r = 0;
  2114. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2115. memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
  2116. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2117. return r;
  2118. }
  2119. static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
  2120. {
  2121. int r = 0;
  2122. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2123. memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
  2124. kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
  2125. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2126. return r;
  2127. }
  2128. static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
  2129. {
  2130. int r = 0;
  2131. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2132. memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
  2133. sizeof(ps->channels));
  2134. ps->flags = kvm->arch.vpit->pit_state.flags;
  2135. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2136. return r;
  2137. }
  2138. static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
  2139. {
  2140. int r = 0, start = 0;
  2141. u32 prev_legacy, cur_legacy;
  2142. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2143. prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
  2144. cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
  2145. if (!prev_legacy && cur_legacy)
  2146. start = 1;
  2147. memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
  2148. sizeof(kvm->arch.vpit->pit_state.channels));
  2149. kvm->arch.vpit->pit_state.flags = ps->flags;
  2150. kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
  2151. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2152. return r;
  2153. }
  2154. static int kvm_vm_ioctl_reinject(struct kvm *kvm,
  2155. struct kvm_reinject_control *control)
  2156. {
  2157. if (!kvm->arch.vpit)
  2158. return -ENXIO;
  2159. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2160. kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject;
  2161. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2162. return 0;
  2163. }
  2164. /*
  2165. * Get (and clear) the dirty memory log for a memory slot.
  2166. */
  2167. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
  2168. struct kvm_dirty_log *log)
  2169. {
  2170. int r, n, i;
  2171. struct kvm_memory_slot *memslot;
  2172. unsigned long is_dirty = 0;
  2173. unsigned long *dirty_bitmap = NULL;
  2174. mutex_lock(&kvm->slots_lock);
  2175. r = -EINVAL;
  2176. if (log->slot >= KVM_MEMORY_SLOTS)
  2177. goto out;
  2178. memslot = &kvm->memslots->memslots[log->slot];
  2179. r = -ENOENT;
  2180. if (!memslot->dirty_bitmap)
  2181. goto out;
  2182. n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
  2183. r = -ENOMEM;
  2184. dirty_bitmap = vmalloc(n);
  2185. if (!dirty_bitmap)
  2186. goto out;
  2187. memset(dirty_bitmap, 0, n);
  2188. for (i = 0; !is_dirty && i < n/sizeof(long); i++)
  2189. is_dirty = memslot->dirty_bitmap[i];
  2190. /* If nothing is dirty, don't bother messing with page tables. */
  2191. if (is_dirty) {
  2192. struct kvm_memslots *slots, *old_slots;
  2193. spin_lock(&kvm->mmu_lock);
  2194. kvm_mmu_slot_remove_write_access(kvm, log->slot);
  2195. spin_unlock(&kvm->mmu_lock);
  2196. slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
  2197. if (!slots)
  2198. goto out_free;
  2199. memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
  2200. slots->memslots[log->slot].dirty_bitmap = dirty_bitmap;
  2201. old_slots = kvm->memslots;
  2202. rcu_assign_pointer(kvm->memslots, slots);
  2203. synchronize_srcu_expedited(&kvm->srcu);
  2204. dirty_bitmap = old_slots->memslots[log->slot].dirty_bitmap;
  2205. kfree(old_slots);
  2206. }
  2207. r = 0;
  2208. if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n))
  2209. r = -EFAULT;
  2210. out_free:
  2211. vfree(dirty_bitmap);
  2212. out:
  2213. mutex_unlock(&kvm->slots_lock);
  2214. return r;
  2215. }
  2216. long kvm_arch_vm_ioctl(struct file *filp,
  2217. unsigned int ioctl, unsigned long arg)
  2218. {
  2219. struct kvm *kvm = filp->private_data;
  2220. void __user *argp = (void __user *)arg;
  2221. int r = -ENOTTY;
  2222. /*
  2223. * This union makes it completely explicit to gcc-3.x
  2224. * that these two variables' stack usage should be
  2225. * combined, not added together.
  2226. */
  2227. union {
  2228. struct kvm_pit_state ps;
  2229. struct kvm_pit_state2 ps2;
  2230. struct kvm_memory_alias alias;
  2231. struct kvm_pit_config pit_config;
  2232. } u;
  2233. switch (ioctl) {
  2234. case KVM_SET_TSS_ADDR:
  2235. r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
  2236. if (r < 0)
  2237. goto out;
  2238. break;
  2239. case KVM_SET_IDENTITY_MAP_ADDR: {
  2240. u64 ident_addr;
  2241. r = -EFAULT;
  2242. if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
  2243. goto out;
  2244. r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
  2245. if (r < 0)
  2246. goto out;
  2247. break;
  2248. }
  2249. case KVM_SET_MEMORY_REGION: {
  2250. struct kvm_memory_region kvm_mem;
  2251. struct kvm_userspace_memory_region kvm_userspace_mem;
  2252. r = -EFAULT;
  2253. if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
  2254. goto out;
  2255. kvm_userspace_mem.slot = kvm_mem.slot;
  2256. kvm_userspace_mem.flags = kvm_mem.flags;
  2257. kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr;
  2258. kvm_userspace_mem.memory_size = kvm_mem.memory_size;
  2259. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0);
  2260. if (r)
  2261. goto out;
  2262. break;
  2263. }
  2264. case KVM_SET_NR_MMU_PAGES:
  2265. r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
  2266. if (r)
  2267. goto out;
  2268. break;
  2269. case KVM_GET_NR_MMU_PAGES:
  2270. r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
  2271. break;
  2272. case KVM_SET_MEMORY_ALIAS:
  2273. r = -EFAULT;
  2274. if (copy_from_user(&u.alias, argp, sizeof(struct kvm_memory_alias)))
  2275. goto out;
  2276. r = kvm_vm_ioctl_set_memory_alias(kvm, &u.alias);
  2277. if (r)
  2278. goto out;
  2279. break;
  2280. case KVM_CREATE_IRQCHIP: {
  2281. struct kvm_pic *vpic;
  2282. mutex_lock(&kvm->lock);
  2283. r = -EEXIST;
  2284. if (kvm->arch.vpic)
  2285. goto create_irqchip_unlock;
  2286. r = -ENOMEM;
  2287. vpic = kvm_create_pic(kvm);
  2288. if (vpic) {
  2289. r = kvm_ioapic_init(kvm);
  2290. if (r) {
  2291. kfree(vpic);
  2292. goto create_irqchip_unlock;
  2293. }
  2294. } else
  2295. goto create_irqchip_unlock;
  2296. smp_wmb();
  2297. kvm->arch.vpic = vpic;
  2298. smp_wmb();
  2299. r = kvm_setup_default_irq_routing(kvm);
  2300. if (r) {
  2301. mutex_lock(&kvm->irq_lock);
  2302. kfree(kvm->arch.vpic);
  2303. kfree(kvm->arch.vioapic);
  2304. kvm->arch.vpic = NULL;
  2305. kvm->arch.vioapic = NULL;
  2306. mutex_unlock(&kvm->irq_lock);
  2307. }
  2308. create_irqchip_unlock:
  2309. mutex_unlock(&kvm->lock);
  2310. break;
  2311. }
  2312. case KVM_CREATE_PIT:
  2313. u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
  2314. goto create_pit;
  2315. case KVM_CREATE_PIT2:
  2316. r = -EFAULT;
  2317. if (copy_from_user(&u.pit_config, argp,
  2318. sizeof(struct kvm_pit_config)))
  2319. goto out;
  2320. create_pit:
  2321. mutex_lock(&kvm->slots_lock);
  2322. r = -EEXIST;
  2323. if (kvm->arch.vpit)
  2324. goto create_pit_unlock;
  2325. r = -ENOMEM;
  2326. kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
  2327. if (kvm->arch.vpit)
  2328. r = 0;
  2329. create_pit_unlock:
  2330. mutex_unlock(&kvm->slots_lock);
  2331. break;
  2332. case KVM_IRQ_LINE_STATUS:
  2333. case KVM_IRQ_LINE: {
  2334. struct kvm_irq_level irq_event;
  2335. r = -EFAULT;
  2336. if (copy_from_user(&irq_event, argp, sizeof irq_event))
  2337. goto out;
  2338. if (irqchip_in_kernel(kvm)) {
  2339. __s32 status;
  2340. status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
  2341. irq_event.irq, irq_event.level);
  2342. if (ioctl == KVM_IRQ_LINE_STATUS) {
  2343. irq_event.status = status;
  2344. if (copy_to_user(argp, &irq_event,
  2345. sizeof irq_event))
  2346. goto out;
  2347. }
  2348. r = 0;
  2349. }
  2350. break;
  2351. }
  2352. case KVM_GET_IRQCHIP: {
  2353. /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
  2354. struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
  2355. r = -ENOMEM;
  2356. if (!chip)
  2357. goto out;
  2358. r = -EFAULT;
  2359. if (copy_from_user(chip, argp, sizeof *chip))
  2360. goto get_irqchip_out;
  2361. r = -ENXIO;
  2362. if (!irqchip_in_kernel(kvm))
  2363. goto get_irqchip_out;
  2364. r = kvm_vm_ioctl_get_irqchip(kvm, chip);
  2365. if (r)
  2366. goto get_irqchip_out;
  2367. r = -EFAULT;
  2368. if (copy_to_user(argp, chip, sizeof *chip))
  2369. goto get_irqchip_out;
  2370. r = 0;
  2371. get_irqchip_out:
  2372. kfree(chip);
  2373. if (r)
  2374. goto out;
  2375. break;
  2376. }
  2377. case KVM_SET_IRQCHIP: {
  2378. /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
  2379. struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
  2380. r = -ENOMEM;
  2381. if (!chip)
  2382. goto out;
  2383. r = -EFAULT;
  2384. if (copy_from_user(chip, argp, sizeof *chip))
  2385. goto set_irqchip_out;
  2386. r = -ENXIO;
  2387. if (!irqchip_in_kernel(kvm))
  2388. goto set_irqchip_out;
  2389. r = kvm_vm_ioctl_set_irqchip(kvm, chip);
  2390. if (r)
  2391. goto set_irqchip_out;
  2392. r = 0;
  2393. set_irqchip_out:
  2394. kfree(chip);
  2395. if (r)
  2396. goto out;
  2397. break;
  2398. }
  2399. case KVM_GET_PIT: {
  2400. r = -EFAULT;
  2401. if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
  2402. goto out;
  2403. r = -ENXIO;
  2404. if (!kvm->arch.vpit)
  2405. goto out;
  2406. r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
  2407. if (r)
  2408. goto out;
  2409. r = -EFAULT;
  2410. if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
  2411. goto out;
  2412. r = 0;
  2413. break;
  2414. }
  2415. case KVM_SET_PIT: {
  2416. r = -EFAULT;
  2417. if (copy_from_user(&u.ps, argp, sizeof u.ps))
  2418. goto out;
  2419. r = -ENXIO;
  2420. if (!kvm->arch.vpit)
  2421. goto out;
  2422. r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
  2423. if (r)
  2424. goto out;
  2425. r = 0;
  2426. break;
  2427. }
  2428. case KVM_GET_PIT2: {
  2429. r = -ENXIO;
  2430. if (!kvm->arch.vpit)
  2431. goto out;
  2432. r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
  2433. if (r)
  2434. goto out;
  2435. r = -EFAULT;
  2436. if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
  2437. goto out;
  2438. r = 0;
  2439. break;
  2440. }
  2441. case KVM_SET_PIT2: {
  2442. r = -EFAULT;
  2443. if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
  2444. goto out;
  2445. r = -ENXIO;
  2446. if (!kvm->arch.vpit)
  2447. goto out;
  2448. r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
  2449. if (r)
  2450. goto out;
  2451. r = 0;
  2452. break;
  2453. }
  2454. case KVM_REINJECT_CONTROL: {
  2455. struct kvm_reinject_control control;
  2456. r = -EFAULT;
  2457. if (copy_from_user(&control, argp, sizeof(control)))
  2458. goto out;
  2459. r = kvm_vm_ioctl_reinject(kvm, &control);
  2460. if (r)
  2461. goto out;
  2462. r = 0;
  2463. break;
  2464. }
  2465. case KVM_XEN_HVM_CONFIG: {
  2466. r = -EFAULT;
  2467. if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
  2468. sizeof(struct kvm_xen_hvm_config)))
  2469. goto out;
  2470. r = -EINVAL;
  2471. if (kvm->arch.xen_hvm_config.flags)
  2472. goto out;
  2473. r = 0;
  2474. break;
  2475. }
  2476. case KVM_SET_CLOCK: {
  2477. struct timespec now;
  2478. struct kvm_clock_data user_ns;
  2479. u64 now_ns;
  2480. s64 delta;
  2481. r = -EFAULT;
  2482. if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
  2483. goto out;
  2484. r = -EINVAL;
  2485. if (user_ns.flags)
  2486. goto out;
  2487. r = 0;
  2488. ktime_get_ts(&now);
  2489. now_ns = timespec_to_ns(&now);
  2490. delta = user_ns.clock - now_ns;
  2491. kvm->arch.kvmclock_offset = delta;
  2492. break;
  2493. }
  2494. case KVM_GET_CLOCK: {
  2495. struct timespec now;
  2496. struct kvm_clock_data user_ns;
  2497. u64 now_ns;
  2498. ktime_get_ts(&now);
  2499. now_ns = timespec_to_ns(&now);
  2500. user_ns.clock = kvm->arch.kvmclock_offset + now_ns;
  2501. user_ns.flags = 0;
  2502. r = -EFAULT;
  2503. if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
  2504. goto out;
  2505. r = 0;
  2506. break;
  2507. }
  2508. default:
  2509. ;
  2510. }
  2511. out:
  2512. return r;
  2513. }
  2514. static void kvm_init_msr_list(void)
  2515. {
  2516. u32 dummy[2];
  2517. unsigned i, j;
  2518. /* skip the first msrs in the list. KVM-specific */
  2519. for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) {
  2520. if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
  2521. continue;
  2522. if (j < i)
  2523. msrs_to_save[j] = msrs_to_save[i];
  2524. j++;
  2525. }
  2526. num_msrs_to_save = j;
  2527. }
  2528. static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
  2529. const void *v)
  2530. {
  2531. if (vcpu->arch.apic &&
  2532. !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, len, v))
  2533. return 0;
  2534. return kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, len, v);
  2535. }
  2536. static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
  2537. {
  2538. if (vcpu->arch.apic &&
  2539. !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, len, v))
  2540. return 0;
  2541. return kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, len, v);
  2542. }
  2543. static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes,
  2544. struct kvm_vcpu *vcpu)
  2545. {
  2546. void *data = val;
  2547. int r = X86EMUL_CONTINUE;
  2548. while (bytes) {
  2549. gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
  2550. unsigned offset = addr & (PAGE_SIZE-1);
  2551. unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
  2552. int ret;
  2553. if (gpa == UNMAPPED_GVA) {
  2554. r = X86EMUL_PROPAGATE_FAULT;
  2555. goto out;
  2556. }
  2557. ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
  2558. if (ret < 0) {
  2559. r = X86EMUL_UNHANDLEABLE;
  2560. goto out;
  2561. }
  2562. bytes -= toread;
  2563. data += toread;
  2564. addr += toread;
  2565. }
  2566. out:
  2567. return r;
  2568. }
  2569. static int kvm_write_guest_virt(gva_t addr, void *val, unsigned int bytes,
  2570. struct kvm_vcpu *vcpu)
  2571. {
  2572. void *data = val;
  2573. int r = X86EMUL_CONTINUE;
  2574. while (bytes) {
  2575. gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
  2576. unsigned offset = addr & (PAGE_SIZE-1);
  2577. unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
  2578. int ret;
  2579. if (gpa == UNMAPPED_GVA) {
  2580. r = X86EMUL_PROPAGATE_FAULT;
  2581. goto out;
  2582. }
  2583. ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
  2584. if (ret < 0) {
  2585. r = X86EMUL_UNHANDLEABLE;
  2586. goto out;
  2587. }
  2588. bytes -= towrite;
  2589. data += towrite;
  2590. addr += towrite;
  2591. }
  2592. out:
  2593. return r;
  2594. }
  2595. static int emulator_read_emulated(unsigned long addr,
  2596. void *val,
  2597. unsigned int bytes,
  2598. struct kvm_vcpu *vcpu)
  2599. {
  2600. gpa_t gpa;
  2601. if (vcpu->mmio_read_completed) {
  2602. memcpy(val, vcpu->mmio_data, bytes);
  2603. trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
  2604. vcpu->mmio_phys_addr, *(u64 *)val);
  2605. vcpu->mmio_read_completed = 0;
  2606. return X86EMUL_CONTINUE;
  2607. }
  2608. gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
  2609. /* For APIC access vmexit */
  2610. if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  2611. goto mmio;
  2612. if (kvm_read_guest_virt(addr, val, bytes, vcpu)
  2613. == X86EMUL_CONTINUE)
  2614. return X86EMUL_CONTINUE;
  2615. if (gpa == UNMAPPED_GVA)
  2616. return X86EMUL_PROPAGATE_FAULT;
  2617. mmio:
  2618. /*
  2619. * Is this MMIO handled locally?
  2620. */
  2621. if (!vcpu_mmio_read(vcpu, gpa, bytes, val)) {
  2622. trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes, gpa, *(u64 *)val);
  2623. return X86EMUL_CONTINUE;
  2624. }
  2625. trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
  2626. vcpu->mmio_needed = 1;
  2627. vcpu->mmio_phys_addr = gpa;
  2628. vcpu->mmio_size = bytes;
  2629. vcpu->mmio_is_write = 0;
  2630. return X86EMUL_UNHANDLEABLE;
  2631. }
  2632. int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
  2633. const void *val, int bytes)
  2634. {
  2635. int ret;
  2636. ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
  2637. if (ret < 0)
  2638. return 0;
  2639. kvm_mmu_pte_write(vcpu, gpa, val, bytes, 1);
  2640. return 1;
  2641. }
  2642. static int emulator_write_emulated_onepage(unsigned long addr,
  2643. const void *val,
  2644. unsigned int bytes,
  2645. struct kvm_vcpu *vcpu)
  2646. {
  2647. gpa_t gpa;
  2648. gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
  2649. if (gpa == UNMAPPED_GVA) {
  2650. kvm_inject_page_fault(vcpu, addr, 2);
  2651. return X86EMUL_PROPAGATE_FAULT;
  2652. }
  2653. /* For APIC access vmexit */
  2654. if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  2655. goto mmio;
  2656. if (emulator_write_phys(vcpu, gpa, val, bytes))
  2657. return X86EMUL_CONTINUE;
  2658. mmio:
  2659. trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
  2660. /*
  2661. * Is this MMIO handled locally?
  2662. */
  2663. if (!vcpu_mmio_write(vcpu, gpa, bytes, val))
  2664. return X86EMUL_CONTINUE;
  2665. vcpu->mmio_needed = 1;
  2666. vcpu->mmio_phys_addr = gpa;
  2667. vcpu->mmio_size = bytes;
  2668. vcpu->mmio_is_write = 1;
  2669. memcpy(vcpu->mmio_data, val, bytes);
  2670. return X86EMUL_CONTINUE;
  2671. }
  2672. int emulator_write_emulated(unsigned long addr,
  2673. const void *val,
  2674. unsigned int bytes,
  2675. struct kvm_vcpu *vcpu)
  2676. {
  2677. /* Crossing a page boundary? */
  2678. if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
  2679. int rc, now;
  2680. now = -addr & ~PAGE_MASK;
  2681. rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
  2682. if (rc != X86EMUL_CONTINUE)
  2683. return rc;
  2684. addr += now;
  2685. val += now;
  2686. bytes -= now;
  2687. }
  2688. return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
  2689. }
  2690. EXPORT_SYMBOL_GPL(emulator_write_emulated);
  2691. static int emulator_cmpxchg_emulated(unsigned long addr,
  2692. const void *old,
  2693. const void *new,
  2694. unsigned int bytes,
  2695. struct kvm_vcpu *vcpu)
  2696. {
  2697. printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
  2698. #ifndef CONFIG_X86_64
  2699. /* guests cmpxchg8b have to be emulated atomically */
  2700. if (bytes == 8) {
  2701. gpa_t gpa;
  2702. struct page *page;
  2703. char *kaddr;
  2704. u64 val;
  2705. gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
  2706. if (gpa == UNMAPPED_GVA ||
  2707. (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  2708. goto emul_write;
  2709. if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
  2710. goto emul_write;
  2711. val = *(u64 *)new;
  2712. page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  2713. kaddr = kmap_atomic(page, KM_USER0);
  2714. set_64bit((u64 *)(kaddr + offset_in_page(gpa)), val);
  2715. kunmap_atomic(kaddr, KM_USER0);
  2716. kvm_release_page_dirty(page);
  2717. }
  2718. emul_write:
  2719. #endif
  2720. return emulator_write_emulated(addr, new, bytes, vcpu);
  2721. }
  2722. static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
  2723. {
  2724. return kvm_x86_ops->get_segment_base(vcpu, seg);
  2725. }
  2726. int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
  2727. {
  2728. kvm_mmu_invlpg(vcpu, address);
  2729. return X86EMUL_CONTINUE;
  2730. }
  2731. int emulate_clts(struct kvm_vcpu *vcpu)
  2732. {
  2733. kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS);
  2734. return X86EMUL_CONTINUE;
  2735. }
  2736. int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
  2737. {
  2738. struct kvm_vcpu *vcpu = ctxt->vcpu;
  2739. switch (dr) {
  2740. case 0 ... 3:
  2741. *dest = kvm_x86_ops->get_dr(vcpu, dr);
  2742. return X86EMUL_CONTINUE;
  2743. default:
  2744. pr_unimpl(vcpu, "%s: unexpected dr %u\n", __func__, dr);
  2745. return X86EMUL_UNHANDLEABLE;
  2746. }
  2747. }
  2748. int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
  2749. {
  2750. unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
  2751. int exception;
  2752. kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
  2753. if (exception) {
  2754. /* FIXME: better handling */
  2755. return X86EMUL_UNHANDLEABLE;
  2756. }
  2757. return X86EMUL_CONTINUE;
  2758. }
  2759. void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context)
  2760. {
  2761. u8 opcodes[4];
  2762. unsigned long rip = kvm_rip_read(vcpu);
  2763. unsigned long rip_linear;
  2764. if (!printk_ratelimit())
  2765. return;
  2766. rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS);
  2767. kvm_read_guest_virt(rip_linear, (void *)opcodes, 4, vcpu);
  2768. printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n",
  2769. context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
  2770. }
  2771. EXPORT_SYMBOL_GPL(kvm_report_emulation_failure);
  2772. static struct x86_emulate_ops emulate_ops = {
  2773. .read_std = kvm_read_guest_virt,
  2774. .read_emulated = emulator_read_emulated,
  2775. .write_emulated = emulator_write_emulated,
  2776. .cmpxchg_emulated = emulator_cmpxchg_emulated,
  2777. };
  2778. static void cache_all_regs(struct kvm_vcpu *vcpu)
  2779. {
  2780. kvm_register_read(vcpu, VCPU_REGS_RAX);
  2781. kvm_register_read(vcpu, VCPU_REGS_RSP);
  2782. kvm_register_read(vcpu, VCPU_REGS_RIP);
  2783. vcpu->arch.regs_dirty = ~0;
  2784. }
  2785. int emulate_instruction(struct kvm_vcpu *vcpu,
  2786. unsigned long cr2,
  2787. u16 error_code,
  2788. int emulation_type)
  2789. {
  2790. int r, shadow_mask;
  2791. struct decode_cache *c;
  2792. struct kvm_run *run = vcpu->run;
  2793. kvm_clear_exception_queue(vcpu);
  2794. vcpu->arch.mmio_fault_cr2 = cr2;
  2795. /*
  2796. * TODO: fix emulate.c to use guest_read/write_register
  2797. * instead of direct ->regs accesses, can save hundred cycles
  2798. * on Intel for instructions that don't read/change RSP, for
  2799. * for example.
  2800. */
  2801. cache_all_regs(vcpu);
  2802. vcpu->mmio_is_write = 0;
  2803. vcpu->arch.pio.string = 0;
  2804. if (!(emulation_type & EMULTYPE_NO_DECODE)) {
  2805. int cs_db, cs_l;
  2806. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  2807. vcpu->arch.emulate_ctxt.vcpu = vcpu;
  2808. vcpu->arch.emulate_ctxt.eflags = kvm_get_rflags(vcpu);
  2809. vcpu->arch.emulate_ctxt.mode =
  2810. (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM)
  2811. ? X86EMUL_MODE_REAL : cs_l
  2812. ? X86EMUL_MODE_PROT64 : cs_db
  2813. ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
  2814. r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
  2815. /* Only allow emulation of specific instructions on #UD
  2816. * (namely VMMCALL, sysenter, sysexit, syscall)*/
  2817. c = &vcpu->arch.emulate_ctxt.decode;
  2818. if (emulation_type & EMULTYPE_TRAP_UD) {
  2819. if (!c->twobyte)
  2820. return EMULATE_FAIL;
  2821. switch (c->b) {
  2822. case 0x01: /* VMMCALL */
  2823. if (c->modrm_mod != 3 || c->modrm_rm != 1)
  2824. return EMULATE_FAIL;
  2825. break;
  2826. case 0x34: /* sysenter */
  2827. case 0x35: /* sysexit */
  2828. if (c->modrm_mod != 0 || c->modrm_rm != 0)
  2829. return EMULATE_FAIL;
  2830. break;
  2831. case 0x05: /* syscall */
  2832. if (c->modrm_mod != 0 || c->modrm_rm != 0)
  2833. return EMULATE_FAIL;
  2834. break;
  2835. default:
  2836. return EMULATE_FAIL;
  2837. }
  2838. if (!(c->modrm_reg == 0 || c->modrm_reg == 3))
  2839. return EMULATE_FAIL;
  2840. }
  2841. ++vcpu->stat.insn_emulation;
  2842. if (r) {
  2843. ++vcpu->stat.insn_emulation_fail;
  2844. if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
  2845. return EMULATE_DONE;
  2846. return EMULATE_FAIL;
  2847. }
  2848. }
  2849. if (emulation_type & EMULTYPE_SKIP) {
  2850. kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.decode.eip);
  2851. return EMULATE_DONE;
  2852. }
  2853. r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
  2854. shadow_mask = vcpu->arch.emulate_ctxt.interruptibility;
  2855. if (r == 0)
  2856. kvm_x86_ops->set_interrupt_shadow(vcpu, shadow_mask);
  2857. if (vcpu->arch.pio.string)
  2858. return EMULATE_DO_MMIO;
  2859. if ((r || vcpu->mmio_is_write) && run) {
  2860. run->exit_reason = KVM_EXIT_MMIO;
  2861. run->mmio.phys_addr = vcpu->mmio_phys_addr;
  2862. memcpy(run->mmio.data, vcpu->mmio_data, 8);
  2863. run->mmio.len = vcpu->mmio_size;
  2864. run->mmio.is_write = vcpu->mmio_is_write;
  2865. }
  2866. if (r) {
  2867. if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
  2868. return EMULATE_DONE;
  2869. if (!vcpu->mmio_needed) {
  2870. kvm_report_emulation_failure(vcpu, "mmio");
  2871. return EMULATE_FAIL;
  2872. }
  2873. return EMULATE_DO_MMIO;
  2874. }
  2875. kvm_set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
  2876. if (vcpu->mmio_is_write) {
  2877. vcpu->mmio_needed = 0;
  2878. return EMULATE_DO_MMIO;
  2879. }
  2880. return EMULATE_DONE;
  2881. }
  2882. EXPORT_SYMBOL_GPL(emulate_instruction);
  2883. static int pio_copy_data(struct kvm_vcpu *vcpu)
  2884. {
  2885. void *p = vcpu->arch.pio_data;
  2886. gva_t q = vcpu->arch.pio.guest_gva;
  2887. unsigned bytes;
  2888. int ret;
  2889. bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count;
  2890. if (vcpu->arch.pio.in)
  2891. ret = kvm_write_guest_virt(q, p, bytes, vcpu);
  2892. else
  2893. ret = kvm_read_guest_virt(q, p, bytes, vcpu);
  2894. return ret;
  2895. }
  2896. int complete_pio(struct kvm_vcpu *vcpu)
  2897. {
  2898. struct kvm_pio_request *io = &vcpu->arch.pio;
  2899. long delta;
  2900. int r;
  2901. unsigned long val;
  2902. if (!io->string) {
  2903. if (io->in) {
  2904. val = kvm_register_read(vcpu, VCPU_REGS_RAX);
  2905. memcpy(&val, vcpu->arch.pio_data, io->size);
  2906. kvm_register_write(vcpu, VCPU_REGS_RAX, val);
  2907. }
  2908. } else {
  2909. if (io->in) {
  2910. r = pio_copy_data(vcpu);
  2911. if (r)
  2912. return r;
  2913. }
  2914. delta = 1;
  2915. if (io->rep) {
  2916. delta *= io->cur_count;
  2917. /*
  2918. * The size of the register should really depend on
  2919. * current address size.
  2920. */
  2921. val = kvm_register_read(vcpu, VCPU_REGS_RCX);
  2922. val -= delta;
  2923. kvm_register_write(vcpu, VCPU_REGS_RCX, val);
  2924. }
  2925. if (io->down)
  2926. delta = -delta;
  2927. delta *= io->size;
  2928. if (io->in) {
  2929. val = kvm_register_read(vcpu, VCPU_REGS_RDI);
  2930. val += delta;
  2931. kvm_register_write(vcpu, VCPU_REGS_RDI, val);
  2932. } else {
  2933. val = kvm_register_read(vcpu, VCPU_REGS_RSI);
  2934. val += delta;
  2935. kvm_register_write(vcpu, VCPU_REGS_RSI, val);
  2936. }
  2937. }
  2938. io->count -= io->cur_count;
  2939. io->cur_count = 0;
  2940. return 0;
  2941. }
  2942. static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
  2943. {
  2944. /* TODO: String I/O for in kernel device */
  2945. int r;
  2946. if (vcpu->arch.pio.in)
  2947. r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
  2948. vcpu->arch.pio.size, pd);
  2949. else
  2950. r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
  2951. vcpu->arch.pio.port, vcpu->arch.pio.size,
  2952. pd);
  2953. return r;
  2954. }
  2955. static int pio_string_write(struct kvm_vcpu *vcpu)
  2956. {
  2957. struct kvm_pio_request *io = &vcpu->arch.pio;
  2958. void *pd = vcpu->arch.pio_data;
  2959. int i, r = 0;
  2960. for (i = 0; i < io->cur_count; i++) {
  2961. if (kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
  2962. io->port, io->size, pd)) {
  2963. r = -EOPNOTSUPP;
  2964. break;
  2965. }
  2966. pd += io->size;
  2967. }
  2968. return r;
  2969. }
  2970. int kvm_emulate_pio(struct kvm_vcpu *vcpu, int in, int size, unsigned port)
  2971. {
  2972. unsigned long val;
  2973. vcpu->run->exit_reason = KVM_EXIT_IO;
  2974. vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  2975. vcpu->run->io.size = vcpu->arch.pio.size = size;
  2976. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  2977. vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1;
  2978. vcpu->run->io.port = vcpu->arch.pio.port = port;
  2979. vcpu->arch.pio.in = in;
  2980. vcpu->arch.pio.string = 0;
  2981. vcpu->arch.pio.down = 0;
  2982. vcpu->arch.pio.rep = 0;
  2983. trace_kvm_pio(vcpu->run->io.direction == KVM_EXIT_IO_OUT, port,
  2984. size, 1);
  2985. val = kvm_register_read(vcpu, VCPU_REGS_RAX);
  2986. memcpy(vcpu->arch.pio_data, &val, 4);
  2987. if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
  2988. complete_pio(vcpu);
  2989. return 1;
  2990. }
  2991. return 0;
  2992. }
  2993. EXPORT_SYMBOL_GPL(kvm_emulate_pio);
  2994. int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, int in,
  2995. int size, unsigned long count, int down,
  2996. gva_t address, int rep, unsigned port)
  2997. {
  2998. unsigned now, in_page;
  2999. int ret = 0;
  3000. vcpu->run->exit_reason = KVM_EXIT_IO;
  3001. vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  3002. vcpu->run->io.size = vcpu->arch.pio.size = size;
  3003. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  3004. vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count;
  3005. vcpu->run->io.port = vcpu->arch.pio.port = port;
  3006. vcpu->arch.pio.in = in;
  3007. vcpu->arch.pio.string = 1;
  3008. vcpu->arch.pio.down = down;
  3009. vcpu->arch.pio.rep = rep;
  3010. trace_kvm_pio(vcpu->run->io.direction == KVM_EXIT_IO_OUT, port,
  3011. size, count);
  3012. if (!count) {
  3013. kvm_x86_ops->skip_emulated_instruction(vcpu);
  3014. return 1;
  3015. }
  3016. if (!down)
  3017. in_page = PAGE_SIZE - offset_in_page(address);
  3018. else
  3019. in_page = offset_in_page(address) + size;
  3020. now = min(count, (unsigned long)in_page / size);
  3021. if (!now)
  3022. now = 1;
  3023. if (down) {
  3024. /*
  3025. * String I/O in reverse. Yuck. Kill the guest, fix later.
  3026. */
  3027. pr_unimpl(vcpu, "guest string pio down\n");
  3028. kvm_inject_gp(vcpu, 0);
  3029. return 1;
  3030. }
  3031. vcpu->run->io.count = now;
  3032. vcpu->arch.pio.cur_count = now;
  3033. if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count)
  3034. kvm_x86_ops->skip_emulated_instruction(vcpu);
  3035. vcpu->arch.pio.guest_gva = address;
  3036. if (!vcpu->arch.pio.in) {
  3037. /* string PIO write */
  3038. ret = pio_copy_data(vcpu);
  3039. if (ret == X86EMUL_PROPAGATE_FAULT) {
  3040. kvm_inject_gp(vcpu, 0);
  3041. return 1;
  3042. }
  3043. if (ret == 0 && !pio_string_write(vcpu)) {
  3044. complete_pio(vcpu);
  3045. if (vcpu->arch.pio.count == 0)
  3046. ret = 1;
  3047. }
  3048. }
  3049. /* no string PIO read support yet */
  3050. return ret;
  3051. }
  3052. EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
  3053. static void bounce_off(void *info)
  3054. {
  3055. /* nothing */
  3056. }
  3057. static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
  3058. void *data)
  3059. {
  3060. struct cpufreq_freqs *freq = data;
  3061. struct kvm *kvm;
  3062. struct kvm_vcpu *vcpu;
  3063. int i, send_ipi = 0;
  3064. if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
  3065. return 0;
  3066. if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
  3067. return 0;
  3068. per_cpu(cpu_tsc_khz, freq->cpu) = freq->new;
  3069. spin_lock(&kvm_lock);
  3070. list_for_each_entry(kvm, &vm_list, vm_list) {
  3071. kvm_for_each_vcpu(i, vcpu, kvm) {
  3072. if (vcpu->cpu != freq->cpu)
  3073. continue;
  3074. if (!kvm_request_guest_time_update(vcpu))
  3075. continue;
  3076. if (vcpu->cpu != smp_processor_id())
  3077. send_ipi++;
  3078. }
  3079. }
  3080. spin_unlock(&kvm_lock);
  3081. if (freq->old < freq->new && send_ipi) {
  3082. /*
  3083. * We upscale the frequency. Must make the guest
  3084. * doesn't see old kvmclock values while running with
  3085. * the new frequency, otherwise we risk the guest sees
  3086. * time go backwards.
  3087. *
  3088. * In case we update the frequency for another cpu
  3089. * (which might be in guest context) send an interrupt
  3090. * to kick the cpu out of guest context. Next time
  3091. * guest context is entered kvmclock will be updated,
  3092. * so the guest will not see stale values.
  3093. */
  3094. smp_call_function_single(freq->cpu, bounce_off, NULL, 1);
  3095. }
  3096. return 0;
  3097. }
  3098. static struct notifier_block kvmclock_cpufreq_notifier_block = {
  3099. .notifier_call = kvmclock_cpufreq_notifier
  3100. };
  3101. static void kvm_timer_init(void)
  3102. {
  3103. int cpu;
  3104. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
  3105. cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
  3106. CPUFREQ_TRANSITION_NOTIFIER);
  3107. for_each_online_cpu(cpu) {
  3108. unsigned long khz = cpufreq_get(cpu);
  3109. if (!khz)
  3110. khz = tsc_khz;
  3111. per_cpu(cpu_tsc_khz, cpu) = khz;
  3112. }
  3113. } else {
  3114. for_each_possible_cpu(cpu)
  3115. per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
  3116. }
  3117. }
  3118. int kvm_arch_init(void *opaque)
  3119. {
  3120. int r;
  3121. struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;
  3122. if (kvm_x86_ops) {
  3123. printk(KERN_ERR "kvm: already loaded the other module\n");
  3124. r = -EEXIST;
  3125. goto out;
  3126. }
  3127. if (!ops->cpu_has_kvm_support()) {
  3128. printk(KERN_ERR "kvm: no hardware support\n");
  3129. r = -EOPNOTSUPP;
  3130. goto out;
  3131. }
  3132. if (ops->disabled_by_bios()) {
  3133. printk(KERN_ERR "kvm: disabled by bios\n");
  3134. r = -EOPNOTSUPP;
  3135. goto out;
  3136. }
  3137. r = kvm_mmu_module_init();
  3138. if (r)
  3139. goto out;
  3140. kvm_init_msr_list();
  3141. kvm_x86_ops = ops;
  3142. kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
  3143. kvm_mmu_set_base_ptes(PT_PRESENT_MASK);
  3144. kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
  3145. PT_DIRTY_MASK, PT64_NX_MASK, 0);
  3146. kvm_timer_init();
  3147. return 0;
  3148. out:
  3149. return r;
  3150. }
  3151. void kvm_arch_exit(void)
  3152. {
  3153. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
  3154. cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
  3155. CPUFREQ_TRANSITION_NOTIFIER);
  3156. kvm_x86_ops = NULL;
  3157. kvm_mmu_module_exit();
  3158. }
  3159. int kvm_emulate_halt(struct kvm_vcpu *vcpu)
  3160. {
  3161. ++vcpu->stat.halt_exits;
  3162. if (irqchip_in_kernel(vcpu->kvm)) {
  3163. vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
  3164. return 1;
  3165. } else {
  3166. vcpu->run->exit_reason = KVM_EXIT_HLT;
  3167. return 0;
  3168. }
  3169. }
  3170. EXPORT_SYMBOL_GPL(kvm_emulate_halt);
  3171. static inline gpa_t hc_gpa(struct kvm_vcpu *vcpu, unsigned long a0,
  3172. unsigned long a1)
  3173. {
  3174. if (is_long_mode(vcpu))
  3175. return a0;
  3176. else
  3177. return a0 | ((gpa_t)a1 << 32);
  3178. }
  3179. int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
  3180. {
  3181. unsigned long nr, a0, a1, a2, a3, ret;
  3182. int r = 1;
  3183. nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3184. a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
  3185. a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3186. a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
  3187. a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
  3188. trace_kvm_hypercall(nr, a0, a1, a2, a3);
  3189. if (!is_long_mode(vcpu)) {
  3190. nr &= 0xFFFFFFFF;
  3191. a0 &= 0xFFFFFFFF;
  3192. a1 &= 0xFFFFFFFF;
  3193. a2 &= 0xFFFFFFFF;
  3194. a3 &= 0xFFFFFFFF;
  3195. }
  3196. if (kvm_x86_ops->get_cpl(vcpu) != 0) {
  3197. ret = -KVM_EPERM;
  3198. goto out;
  3199. }
  3200. switch (nr) {
  3201. case KVM_HC_VAPIC_POLL_IRQ:
  3202. ret = 0;
  3203. break;
  3204. case KVM_HC_MMU_OP:
  3205. r = kvm_pv_mmu_op(vcpu, a0, hc_gpa(vcpu, a1, a2), &ret);
  3206. break;
  3207. default:
  3208. ret = -KVM_ENOSYS;
  3209. break;
  3210. }
  3211. out:
  3212. kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
  3213. ++vcpu->stat.hypercalls;
  3214. return r;
  3215. }
  3216. EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
  3217. int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
  3218. {
  3219. char instruction[3];
  3220. int ret = 0;
  3221. unsigned long rip = kvm_rip_read(vcpu);
  3222. /*
  3223. * Blow out the MMU to ensure that no other VCPU has an active mapping
  3224. * to ensure that the updated hypercall appears atomically across all
  3225. * VCPUs.
  3226. */
  3227. kvm_mmu_zap_all(vcpu->kvm);
  3228. kvm_x86_ops->patch_hypercall(vcpu, instruction);
  3229. if (emulator_write_emulated(rip, instruction, 3, vcpu)
  3230. != X86EMUL_CONTINUE)
  3231. ret = -EFAULT;
  3232. return ret;
  3233. }
  3234. static u64 mk_cr_64(u64 curr_cr, u32 new_val)
  3235. {
  3236. return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
  3237. }
  3238. void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  3239. {
  3240. struct descriptor_table dt = { limit, base };
  3241. kvm_x86_ops->set_gdt(vcpu, &dt);
  3242. }
  3243. void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  3244. {
  3245. struct descriptor_table dt = { limit, base };
  3246. kvm_x86_ops->set_idt(vcpu, &dt);
  3247. }
  3248. void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
  3249. unsigned long *rflags)
  3250. {
  3251. kvm_lmsw(vcpu, msw);
  3252. *rflags = kvm_get_rflags(vcpu);
  3253. }
  3254. unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
  3255. {
  3256. unsigned long value;
  3257. switch (cr) {
  3258. case 0:
  3259. value = vcpu->arch.cr0;
  3260. break;
  3261. case 2:
  3262. value = vcpu->arch.cr2;
  3263. break;
  3264. case 3:
  3265. value = vcpu->arch.cr3;
  3266. break;
  3267. case 4:
  3268. value = kvm_read_cr4(vcpu);
  3269. break;
  3270. case 8:
  3271. value = kvm_get_cr8(vcpu);
  3272. break;
  3273. default:
  3274. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
  3275. return 0;
  3276. }
  3277. return value;
  3278. }
  3279. void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
  3280. unsigned long *rflags)
  3281. {
  3282. switch (cr) {
  3283. case 0:
  3284. kvm_set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val));
  3285. *rflags = kvm_get_rflags(vcpu);
  3286. break;
  3287. case 2:
  3288. vcpu->arch.cr2 = val;
  3289. break;
  3290. case 3:
  3291. kvm_set_cr3(vcpu, val);
  3292. break;
  3293. case 4:
  3294. kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
  3295. break;
  3296. case 8:
  3297. kvm_set_cr8(vcpu, val & 0xfUL);
  3298. break;
  3299. default:
  3300. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
  3301. }
  3302. }
  3303. static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
  3304. {
  3305. struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
  3306. int j, nent = vcpu->arch.cpuid_nent;
  3307. e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
  3308. /* when no next entry is found, the current entry[i] is reselected */
  3309. for (j = i + 1; ; j = (j + 1) % nent) {
  3310. struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j];
  3311. if (ej->function == e->function) {
  3312. ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
  3313. return j;
  3314. }
  3315. }
  3316. return 0; /* silence gcc, even though control never reaches here */
  3317. }
  3318. /* find an entry with matching function, matching index (if needed), and that
  3319. * should be read next (if it's stateful) */
  3320. static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
  3321. u32 function, u32 index)
  3322. {
  3323. if (e->function != function)
  3324. return 0;
  3325. if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
  3326. return 0;
  3327. if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
  3328. !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
  3329. return 0;
  3330. return 1;
  3331. }
  3332. struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
  3333. u32 function, u32 index)
  3334. {
  3335. int i;
  3336. struct kvm_cpuid_entry2 *best = NULL;
  3337. for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
  3338. struct kvm_cpuid_entry2 *e;
  3339. e = &vcpu->arch.cpuid_entries[i];
  3340. if (is_matching_cpuid_entry(e, function, index)) {
  3341. if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
  3342. move_to_next_stateful_cpuid_entry(vcpu, i);
  3343. best = e;
  3344. break;
  3345. }
  3346. /*
  3347. * Both basic or both extended?
  3348. */
  3349. if (((e->function ^ function) & 0x80000000) == 0)
  3350. if (!best || e->function > best->function)
  3351. best = e;
  3352. }
  3353. return best;
  3354. }
  3355. EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
  3356. int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
  3357. {
  3358. struct kvm_cpuid_entry2 *best;
  3359. best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
  3360. if (best)
  3361. return best->eax & 0xff;
  3362. return 36;
  3363. }
  3364. void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
  3365. {
  3366. u32 function, index;
  3367. struct kvm_cpuid_entry2 *best;
  3368. function = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3369. index = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3370. kvm_register_write(vcpu, VCPU_REGS_RAX, 0);
  3371. kvm_register_write(vcpu, VCPU_REGS_RBX, 0);
  3372. kvm_register_write(vcpu, VCPU_REGS_RCX, 0);
  3373. kvm_register_write(vcpu, VCPU_REGS_RDX, 0);
  3374. best = kvm_find_cpuid_entry(vcpu, function, index);
  3375. if (best) {
  3376. kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax);
  3377. kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx);
  3378. kvm_register_write(vcpu, VCPU_REGS_RCX, best->ecx);
  3379. kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx);
  3380. }
  3381. kvm_x86_ops->skip_emulated_instruction(vcpu);
  3382. trace_kvm_cpuid(function,
  3383. kvm_register_read(vcpu, VCPU_REGS_RAX),
  3384. kvm_register_read(vcpu, VCPU_REGS_RBX),
  3385. kvm_register_read(vcpu, VCPU_REGS_RCX),
  3386. kvm_register_read(vcpu, VCPU_REGS_RDX));
  3387. }
  3388. EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
  3389. /*
  3390. * Check if userspace requested an interrupt window, and that the
  3391. * interrupt window is open.
  3392. *
  3393. * No need to exit to userspace if we already have an interrupt queued.
  3394. */
  3395. static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
  3396. {
  3397. return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
  3398. vcpu->run->request_interrupt_window &&
  3399. kvm_arch_interrupt_allowed(vcpu));
  3400. }
  3401. static void post_kvm_run_save(struct kvm_vcpu *vcpu)
  3402. {
  3403. struct kvm_run *kvm_run = vcpu->run;
  3404. kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
  3405. kvm_run->cr8 = kvm_get_cr8(vcpu);
  3406. kvm_run->apic_base = kvm_get_apic_base(vcpu);
  3407. if (irqchip_in_kernel(vcpu->kvm))
  3408. kvm_run->ready_for_interrupt_injection = 1;
  3409. else
  3410. kvm_run->ready_for_interrupt_injection =
  3411. kvm_arch_interrupt_allowed(vcpu) &&
  3412. !kvm_cpu_has_interrupt(vcpu) &&
  3413. !kvm_event_needs_reinjection(vcpu);
  3414. }
  3415. static void vapic_enter(struct kvm_vcpu *vcpu)
  3416. {
  3417. struct kvm_lapic *apic = vcpu->arch.apic;
  3418. struct page *page;
  3419. if (!apic || !apic->vapic_addr)
  3420. return;
  3421. page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
  3422. vcpu->arch.apic->vapic_page = page;
  3423. }
  3424. static void vapic_exit(struct kvm_vcpu *vcpu)
  3425. {
  3426. struct kvm_lapic *apic = vcpu->arch.apic;
  3427. int idx;
  3428. if (!apic || !apic->vapic_addr)
  3429. return;
  3430. idx = srcu_read_lock(&vcpu->kvm->srcu);
  3431. kvm_release_page_dirty(apic->vapic_page);
  3432. mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
  3433. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  3434. }
  3435. static void update_cr8_intercept(struct kvm_vcpu *vcpu)
  3436. {
  3437. int max_irr, tpr;
  3438. if (!kvm_x86_ops->update_cr8_intercept)
  3439. return;
  3440. if (!vcpu->arch.apic)
  3441. return;
  3442. if (!vcpu->arch.apic->vapic_addr)
  3443. max_irr = kvm_lapic_find_highest_irr(vcpu);
  3444. else
  3445. max_irr = -1;
  3446. if (max_irr != -1)
  3447. max_irr >>= 4;
  3448. tpr = kvm_lapic_get_cr8(vcpu);
  3449. kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
  3450. }
  3451. static void inject_pending_event(struct kvm_vcpu *vcpu)
  3452. {
  3453. /* try to reinject previous events if any */
  3454. if (vcpu->arch.exception.pending) {
  3455. kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
  3456. vcpu->arch.exception.has_error_code,
  3457. vcpu->arch.exception.error_code);
  3458. return;
  3459. }
  3460. if (vcpu->arch.nmi_injected) {
  3461. kvm_x86_ops->set_nmi(vcpu);
  3462. return;
  3463. }
  3464. if (vcpu->arch.interrupt.pending) {
  3465. kvm_x86_ops->set_irq(vcpu);
  3466. return;
  3467. }
  3468. /* try to inject new event if pending */
  3469. if (vcpu->arch.nmi_pending) {
  3470. if (kvm_x86_ops->nmi_allowed(vcpu)) {
  3471. vcpu->arch.nmi_pending = false;
  3472. vcpu->arch.nmi_injected = true;
  3473. kvm_x86_ops->set_nmi(vcpu);
  3474. }
  3475. } else if (kvm_cpu_has_interrupt(vcpu)) {
  3476. if (kvm_x86_ops->interrupt_allowed(vcpu)) {
  3477. kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
  3478. false);
  3479. kvm_x86_ops->set_irq(vcpu);
  3480. }
  3481. }
  3482. }
  3483. static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
  3484. {
  3485. int r;
  3486. bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
  3487. vcpu->run->request_interrupt_window;
  3488. if (vcpu->requests)
  3489. if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
  3490. kvm_mmu_unload(vcpu);
  3491. r = kvm_mmu_reload(vcpu);
  3492. if (unlikely(r))
  3493. goto out;
  3494. if (vcpu->requests) {
  3495. if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests))
  3496. __kvm_migrate_timers(vcpu);
  3497. if (test_and_clear_bit(KVM_REQ_KVMCLOCK_UPDATE, &vcpu->requests))
  3498. kvm_write_guest_time(vcpu);
  3499. if (test_and_clear_bit(KVM_REQ_MMU_SYNC, &vcpu->requests))
  3500. kvm_mmu_sync_roots(vcpu);
  3501. if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
  3502. kvm_x86_ops->tlb_flush(vcpu);
  3503. if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS,
  3504. &vcpu->requests)) {
  3505. vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
  3506. r = 0;
  3507. goto out;
  3508. }
  3509. if (test_and_clear_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests)) {
  3510. vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
  3511. r = 0;
  3512. goto out;
  3513. }
  3514. }
  3515. preempt_disable();
  3516. kvm_x86_ops->prepare_guest_switch(vcpu);
  3517. kvm_load_guest_fpu(vcpu);
  3518. local_irq_disable();
  3519. clear_bit(KVM_REQ_KICK, &vcpu->requests);
  3520. smp_mb__after_clear_bit();
  3521. if (vcpu->requests || need_resched() || signal_pending(current)) {
  3522. set_bit(KVM_REQ_KICK, &vcpu->requests);
  3523. local_irq_enable();
  3524. preempt_enable();
  3525. r = 1;
  3526. goto out;
  3527. }
  3528. inject_pending_event(vcpu);
  3529. /* enable NMI/IRQ window open exits if needed */
  3530. if (vcpu->arch.nmi_pending)
  3531. kvm_x86_ops->enable_nmi_window(vcpu);
  3532. else if (kvm_cpu_has_interrupt(vcpu) || req_int_win)
  3533. kvm_x86_ops->enable_irq_window(vcpu);
  3534. if (kvm_lapic_enabled(vcpu)) {
  3535. update_cr8_intercept(vcpu);
  3536. kvm_lapic_sync_to_vapic(vcpu);
  3537. }
  3538. srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
  3539. kvm_guest_enter();
  3540. if (unlikely(vcpu->arch.switch_db_regs)) {
  3541. set_debugreg(0, 7);
  3542. set_debugreg(vcpu->arch.eff_db[0], 0);
  3543. set_debugreg(vcpu->arch.eff_db[1], 1);
  3544. set_debugreg(vcpu->arch.eff_db[2], 2);
  3545. set_debugreg(vcpu->arch.eff_db[3], 3);
  3546. }
  3547. trace_kvm_entry(vcpu->vcpu_id);
  3548. kvm_x86_ops->run(vcpu);
  3549. /*
  3550. * If the guest has used debug registers, at least dr7
  3551. * will be disabled while returning to the host.
  3552. * If we don't have active breakpoints in the host, we don't
  3553. * care about the messed up debug address registers. But if
  3554. * we have some of them active, restore the old state.
  3555. */
  3556. if (hw_breakpoint_active())
  3557. hw_breakpoint_restore();
  3558. set_bit(KVM_REQ_KICK, &vcpu->requests);
  3559. local_irq_enable();
  3560. ++vcpu->stat.exits;
  3561. /*
  3562. * We must have an instruction between local_irq_enable() and
  3563. * kvm_guest_exit(), so the timer interrupt isn't delayed by
  3564. * the interrupt shadow. The stat.exits increment will do nicely.
  3565. * But we need to prevent reordering, hence this barrier():
  3566. */
  3567. barrier();
  3568. kvm_guest_exit();
  3569. preempt_enable();
  3570. vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  3571. /*
  3572. * Profile KVM exit RIPs:
  3573. */
  3574. if (unlikely(prof_on == KVM_PROFILING)) {
  3575. unsigned long rip = kvm_rip_read(vcpu);
  3576. profile_hit(KVM_PROFILING, (void *)rip);
  3577. }
  3578. kvm_lapic_sync_from_vapic(vcpu);
  3579. r = kvm_x86_ops->handle_exit(vcpu);
  3580. out:
  3581. return r;
  3582. }
  3583. static int __vcpu_run(struct kvm_vcpu *vcpu)
  3584. {
  3585. int r;
  3586. struct kvm *kvm = vcpu->kvm;
  3587. if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) {
  3588. pr_debug("vcpu %d received sipi with vector # %x\n",
  3589. vcpu->vcpu_id, vcpu->arch.sipi_vector);
  3590. kvm_lapic_reset(vcpu);
  3591. r = kvm_arch_vcpu_reset(vcpu);
  3592. if (r)
  3593. return r;
  3594. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  3595. }
  3596. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  3597. vapic_enter(vcpu);
  3598. r = 1;
  3599. while (r > 0) {
  3600. if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE)
  3601. r = vcpu_enter_guest(vcpu);
  3602. else {
  3603. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  3604. kvm_vcpu_block(vcpu);
  3605. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  3606. if (test_and_clear_bit(KVM_REQ_UNHALT, &vcpu->requests))
  3607. {
  3608. switch(vcpu->arch.mp_state) {
  3609. case KVM_MP_STATE_HALTED:
  3610. vcpu->arch.mp_state =
  3611. KVM_MP_STATE_RUNNABLE;
  3612. case KVM_MP_STATE_RUNNABLE:
  3613. break;
  3614. case KVM_MP_STATE_SIPI_RECEIVED:
  3615. default:
  3616. r = -EINTR;
  3617. break;
  3618. }
  3619. }
  3620. }
  3621. if (r <= 0)
  3622. break;
  3623. clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
  3624. if (kvm_cpu_has_pending_timer(vcpu))
  3625. kvm_inject_pending_timer_irqs(vcpu);
  3626. if (dm_request_for_irq_injection(vcpu)) {
  3627. r = -EINTR;
  3628. vcpu->run->exit_reason = KVM_EXIT_INTR;
  3629. ++vcpu->stat.request_irq_exits;
  3630. }
  3631. if (signal_pending(current)) {
  3632. r = -EINTR;
  3633. vcpu->run->exit_reason = KVM_EXIT_INTR;
  3634. ++vcpu->stat.signal_exits;
  3635. }
  3636. if (need_resched()) {
  3637. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  3638. kvm_resched(vcpu);
  3639. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  3640. }
  3641. }
  3642. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  3643. post_kvm_run_save(vcpu);
  3644. vapic_exit(vcpu);
  3645. return r;
  3646. }
  3647. int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  3648. {
  3649. int r;
  3650. sigset_t sigsaved;
  3651. vcpu_load(vcpu);
  3652. if (vcpu->sigset_active)
  3653. sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
  3654. if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
  3655. kvm_vcpu_block(vcpu);
  3656. clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
  3657. r = -EAGAIN;
  3658. goto out;
  3659. }
  3660. /* re-sync apic's tpr */
  3661. if (!irqchip_in_kernel(vcpu->kvm))
  3662. kvm_set_cr8(vcpu, kvm_run->cr8);
  3663. if (vcpu->arch.pio.cur_count) {
  3664. r = complete_pio(vcpu);
  3665. if (r)
  3666. goto out;
  3667. }
  3668. if (vcpu->mmio_needed) {
  3669. memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
  3670. vcpu->mmio_read_completed = 1;
  3671. vcpu->mmio_needed = 0;
  3672. vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  3673. r = emulate_instruction(vcpu, vcpu->arch.mmio_fault_cr2, 0,
  3674. EMULTYPE_NO_DECODE);
  3675. srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
  3676. if (r == EMULATE_DO_MMIO) {
  3677. /*
  3678. * Read-modify-write. Back to userspace.
  3679. */
  3680. r = 0;
  3681. goto out;
  3682. }
  3683. }
  3684. if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL)
  3685. kvm_register_write(vcpu, VCPU_REGS_RAX,
  3686. kvm_run->hypercall.ret);
  3687. r = __vcpu_run(vcpu);
  3688. out:
  3689. if (vcpu->sigset_active)
  3690. sigprocmask(SIG_SETMASK, &sigsaved, NULL);
  3691. vcpu_put(vcpu);
  3692. return r;
  3693. }
  3694. int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  3695. {
  3696. vcpu_load(vcpu);
  3697. regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3698. regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  3699. regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3700. regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  3701. regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
  3702. regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
  3703. regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  3704. regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  3705. #ifdef CONFIG_X86_64
  3706. regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
  3707. regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
  3708. regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
  3709. regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
  3710. regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
  3711. regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
  3712. regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
  3713. regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
  3714. #endif
  3715. regs->rip = kvm_rip_read(vcpu);
  3716. regs->rflags = kvm_get_rflags(vcpu);
  3717. vcpu_put(vcpu);
  3718. return 0;
  3719. }
  3720. int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  3721. {
  3722. vcpu_load(vcpu);
  3723. kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
  3724. kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
  3725. kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
  3726. kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
  3727. kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
  3728. kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
  3729. kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
  3730. kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
  3731. #ifdef CONFIG_X86_64
  3732. kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
  3733. kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
  3734. kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
  3735. kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
  3736. kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
  3737. kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
  3738. kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
  3739. kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
  3740. #endif
  3741. kvm_rip_write(vcpu, regs->rip);
  3742. kvm_set_rflags(vcpu, regs->rflags);
  3743. vcpu->arch.exception.pending = false;
  3744. vcpu_put(vcpu);
  3745. return 0;
  3746. }
  3747. void kvm_get_segment(struct kvm_vcpu *vcpu,
  3748. struct kvm_segment *var, int seg)
  3749. {
  3750. kvm_x86_ops->get_segment(vcpu, var, seg);
  3751. }
  3752. void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
  3753. {
  3754. struct kvm_segment cs;
  3755. kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
  3756. *db = cs.db;
  3757. *l = cs.l;
  3758. }
  3759. EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
  3760. int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  3761. struct kvm_sregs *sregs)
  3762. {
  3763. struct descriptor_table dt;
  3764. vcpu_load(vcpu);
  3765. kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  3766. kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  3767. kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  3768. kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  3769. kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  3770. kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  3771. kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  3772. kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  3773. kvm_x86_ops->get_idt(vcpu, &dt);
  3774. sregs->idt.limit = dt.limit;
  3775. sregs->idt.base = dt.base;
  3776. kvm_x86_ops->get_gdt(vcpu, &dt);
  3777. sregs->gdt.limit = dt.limit;
  3778. sregs->gdt.base = dt.base;
  3779. sregs->cr0 = vcpu->arch.cr0;
  3780. sregs->cr2 = vcpu->arch.cr2;
  3781. sregs->cr3 = vcpu->arch.cr3;
  3782. sregs->cr4 = kvm_read_cr4(vcpu);
  3783. sregs->cr8 = kvm_get_cr8(vcpu);
  3784. sregs->efer = vcpu->arch.shadow_efer;
  3785. sregs->apic_base = kvm_get_apic_base(vcpu);
  3786. memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
  3787. if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
  3788. set_bit(vcpu->arch.interrupt.nr,
  3789. (unsigned long *)sregs->interrupt_bitmap);
  3790. vcpu_put(vcpu);
  3791. return 0;
  3792. }
  3793. int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
  3794. struct kvm_mp_state *mp_state)
  3795. {
  3796. vcpu_load(vcpu);
  3797. mp_state->mp_state = vcpu->arch.mp_state;
  3798. vcpu_put(vcpu);
  3799. return 0;
  3800. }
  3801. int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
  3802. struct kvm_mp_state *mp_state)
  3803. {
  3804. vcpu_load(vcpu);
  3805. vcpu->arch.mp_state = mp_state->mp_state;
  3806. vcpu_put(vcpu);
  3807. return 0;
  3808. }
  3809. static void kvm_set_segment(struct kvm_vcpu *vcpu,
  3810. struct kvm_segment *var, int seg)
  3811. {
  3812. kvm_x86_ops->set_segment(vcpu, var, seg);
  3813. }
  3814. static void seg_desct_to_kvm_desct(struct desc_struct *seg_desc, u16 selector,
  3815. struct kvm_segment *kvm_desct)
  3816. {
  3817. kvm_desct->base = get_desc_base(seg_desc);
  3818. kvm_desct->limit = get_desc_limit(seg_desc);
  3819. if (seg_desc->g) {
  3820. kvm_desct->limit <<= 12;
  3821. kvm_desct->limit |= 0xfff;
  3822. }
  3823. kvm_desct->selector = selector;
  3824. kvm_desct->type = seg_desc->type;
  3825. kvm_desct->present = seg_desc->p;
  3826. kvm_desct->dpl = seg_desc->dpl;
  3827. kvm_desct->db = seg_desc->d;
  3828. kvm_desct->s = seg_desc->s;
  3829. kvm_desct->l = seg_desc->l;
  3830. kvm_desct->g = seg_desc->g;
  3831. kvm_desct->avl = seg_desc->avl;
  3832. if (!selector)
  3833. kvm_desct->unusable = 1;
  3834. else
  3835. kvm_desct->unusable = 0;
  3836. kvm_desct->padding = 0;
  3837. }
  3838. static void get_segment_descriptor_dtable(struct kvm_vcpu *vcpu,
  3839. u16 selector,
  3840. struct descriptor_table *dtable)
  3841. {
  3842. if (selector & 1 << 2) {
  3843. struct kvm_segment kvm_seg;
  3844. kvm_get_segment(vcpu, &kvm_seg, VCPU_SREG_LDTR);
  3845. if (kvm_seg.unusable)
  3846. dtable->limit = 0;
  3847. else
  3848. dtable->limit = kvm_seg.limit;
  3849. dtable->base = kvm_seg.base;
  3850. }
  3851. else
  3852. kvm_x86_ops->get_gdt(vcpu, dtable);
  3853. }
  3854. /* allowed just for 8 bytes segments */
  3855. static int load_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
  3856. struct desc_struct *seg_desc)
  3857. {
  3858. struct descriptor_table dtable;
  3859. u16 index = selector >> 3;
  3860. get_segment_descriptor_dtable(vcpu, selector, &dtable);
  3861. if (dtable.limit < index * 8 + 7) {
  3862. kvm_queue_exception_e(vcpu, GP_VECTOR, selector & 0xfffc);
  3863. return 1;
  3864. }
  3865. return kvm_read_guest_virt(dtable.base + index*8, seg_desc, sizeof(*seg_desc), vcpu);
  3866. }
  3867. /* allowed just for 8 bytes segments */
  3868. static int save_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
  3869. struct desc_struct *seg_desc)
  3870. {
  3871. struct descriptor_table dtable;
  3872. u16 index = selector >> 3;
  3873. get_segment_descriptor_dtable(vcpu, selector, &dtable);
  3874. if (dtable.limit < index * 8 + 7)
  3875. return 1;
  3876. return kvm_write_guest_virt(dtable.base + index*8, seg_desc, sizeof(*seg_desc), vcpu);
  3877. }
  3878. static gpa_t get_tss_base_addr(struct kvm_vcpu *vcpu,
  3879. struct desc_struct *seg_desc)
  3880. {
  3881. u32 base_addr = get_desc_base(seg_desc);
  3882. return vcpu->arch.mmu.gva_to_gpa(vcpu, base_addr);
  3883. }
  3884. static u16 get_segment_selector(struct kvm_vcpu *vcpu, int seg)
  3885. {
  3886. struct kvm_segment kvm_seg;
  3887. kvm_get_segment(vcpu, &kvm_seg, seg);
  3888. return kvm_seg.selector;
  3889. }
  3890. static int load_segment_descriptor_to_kvm_desct(struct kvm_vcpu *vcpu,
  3891. u16 selector,
  3892. struct kvm_segment *kvm_seg)
  3893. {
  3894. struct desc_struct seg_desc;
  3895. if (load_guest_segment_descriptor(vcpu, selector, &seg_desc))
  3896. return 1;
  3897. seg_desct_to_kvm_desct(&seg_desc, selector, kvm_seg);
  3898. return 0;
  3899. }
  3900. static int kvm_load_realmode_segment(struct kvm_vcpu *vcpu, u16 selector, int seg)
  3901. {
  3902. struct kvm_segment segvar = {
  3903. .base = selector << 4,
  3904. .limit = 0xffff,
  3905. .selector = selector,
  3906. .type = 3,
  3907. .present = 1,
  3908. .dpl = 3,
  3909. .db = 0,
  3910. .s = 1,
  3911. .l = 0,
  3912. .g = 0,
  3913. .avl = 0,
  3914. .unusable = 0,
  3915. };
  3916. kvm_x86_ops->set_segment(vcpu, &segvar, seg);
  3917. return 0;
  3918. }
  3919. static int is_vm86_segment(struct kvm_vcpu *vcpu, int seg)
  3920. {
  3921. return (seg != VCPU_SREG_LDTR) &&
  3922. (seg != VCPU_SREG_TR) &&
  3923. (kvm_get_rflags(vcpu) & X86_EFLAGS_VM);
  3924. }
  3925. static void kvm_check_segment_descriptor(struct kvm_vcpu *vcpu, int seg,
  3926. u16 selector)
  3927. {
  3928. /* NULL selector is not valid for CS and SS */
  3929. if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
  3930. if (!selector)
  3931. kvm_queue_exception_e(vcpu, TS_VECTOR, selector >> 3);
  3932. }
  3933. int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
  3934. int type_bits, int seg)
  3935. {
  3936. struct kvm_segment kvm_seg;
  3937. if (is_vm86_segment(vcpu, seg) || !(vcpu->arch.cr0 & X86_CR0_PE))
  3938. return kvm_load_realmode_segment(vcpu, selector, seg);
  3939. if (load_segment_descriptor_to_kvm_desct(vcpu, selector, &kvm_seg))
  3940. return 1;
  3941. kvm_check_segment_descriptor(vcpu, seg, selector);
  3942. kvm_seg.type |= type_bits;
  3943. if (seg != VCPU_SREG_SS && seg != VCPU_SREG_CS &&
  3944. seg != VCPU_SREG_LDTR)
  3945. if (!kvm_seg.s)
  3946. kvm_seg.unusable = 1;
  3947. kvm_set_segment(vcpu, &kvm_seg, seg);
  3948. return 0;
  3949. }
  3950. static void save_state_to_tss32(struct kvm_vcpu *vcpu,
  3951. struct tss_segment_32 *tss)
  3952. {
  3953. tss->cr3 = vcpu->arch.cr3;
  3954. tss->eip = kvm_rip_read(vcpu);
  3955. tss->eflags = kvm_get_rflags(vcpu);
  3956. tss->eax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3957. tss->ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3958. tss->edx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  3959. tss->ebx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  3960. tss->esp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  3961. tss->ebp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  3962. tss->esi = kvm_register_read(vcpu, VCPU_REGS_RSI);
  3963. tss->edi = kvm_register_read(vcpu, VCPU_REGS_RDI);
  3964. tss->es = get_segment_selector(vcpu, VCPU_SREG_ES);
  3965. tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS);
  3966. tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS);
  3967. tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS);
  3968. tss->fs = get_segment_selector(vcpu, VCPU_SREG_FS);
  3969. tss->gs = get_segment_selector(vcpu, VCPU_SREG_GS);
  3970. tss->ldt_selector = get_segment_selector(vcpu, VCPU_SREG_LDTR);
  3971. }
  3972. static int load_state_from_tss32(struct kvm_vcpu *vcpu,
  3973. struct tss_segment_32 *tss)
  3974. {
  3975. kvm_set_cr3(vcpu, tss->cr3);
  3976. kvm_rip_write(vcpu, tss->eip);
  3977. kvm_set_rflags(vcpu, tss->eflags | 2);
  3978. kvm_register_write(vcpu, VCPU_REGS_RAX, tss->eax);
  3979. kvm_register_write(vcpu, VCPU_REGS_RCX, tss->ecx);
  3980. kvm_register_write(vcpu, VCPU_REGS_RDX, tss->edx);
  3981. kvm_register_write(vcpu, VCPU_REGS_RBX, tss->ebx);
  3982. kvm_register_write(vcpu, VCPU_REGS_RSP, tss->esp);
  3983. kvm_register_write(vcpu, VCPU_REGS_RBP, tss->ebp);
  3984. kvm_register_write(vcpu, VCPU_REGS_RSI, tss->esi);
  3985. kvm_register_write(vcpu, VCPU_REGS_RDI, tss->edi);
  3986. if (kvm_load_segment_descriptor(vcpu, tss->ldt_selector, 0, VCPU_SREG_LDTR))
  3987. return 1;
  3988. if (kvm_load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES))
  3989. return 1;
  3990. if (kvm_load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS))
  3991. return 1;
  3992. if (kvm_load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS))
  3993. return 1;
  3994. if (kvm_load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS))
  3995. return 1;
  3996. if (kvm_load_segment_descriptor(vcpu, tss->fs, 1, VCPU_SREG_FS))
  3997. return 1;
  3998. if (kvm_load_segment_descriptor(vcpu, tss->gs, 1, VCPU_SREG_GS))
  3999. return 1;
  4000. return 0;
  4001. }
  4002. static void save_state_to_tss16(struct kvm_vcpu *vcpu,
  4003. struct tss_segment_16 *tss)
  4004. {
  4005. tss->ip = kvm_rip_read(vcpu);
  4006. tss->flag = kvm_get_rflags(vcpu);
  4007. tss->ax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  4008. tss->cx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  4009. tss->dx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  4010. tss->bx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  4011. tss->sp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  4012. tss->bp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  4013. tss->si = kvm_register_read(vcpu, VCPU_REGS_RSI);
  4014. tss->di = kvm_register_read(vcpu, VCPU_REGS_RDI);
  4015. tss->es = get_segment_selector(vcpu, VCPU_SREG_ES);
  4016. tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS);
  4017. tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS);
  4018. tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS);
  4019. tss->ldt = get_segment_selector(vcpu, VCPU_SREG_LDTR);
  4020. }
  4021. static int load_state_from_tss16(struct kvm_vcpu *vcpu,
  4022. struct tss_segment_16 *tss)
  4023. {
  4024. kvm_rip_write(vcpu, tss->ip);
  4025. kvm_set_rflags(vcpu, tss->flag | 2);
  4026. kvm_register_write(vcpu, VCPU_REGS_RAX, tss->ax);
  4027. kvm_register_write(vcpu, VCPU_REGS_RCX, tss->cx);
  4028. kvm_register_write(vcpu, VCPU_REGS_RDX, tss->dx);
  4029. kvm_register_write(vcpu, VCPU_REGS_RBX, tss->bx);
  4030. kvm_register_write(vcpu, VCPU_REGS_RSP, tss->sp);
  4031. kvm_register_write(vcpu, VCPU_REGS_RBP, tss->bp);
  4032. kvm_register_write(vcpu, VCPU_REGS_RSI, tss->si);
  4033. kvm_register_write(vcpu, VCPU_REGS_RDI, tss->di);
  4034. if (kvm_load_segment_descriptor(vcpu, tss->ldt, 0, VCPU_SREG_LDTR))
  4035. return 1;
  4036. if (kvm_load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES))
  4037. return 1;
  4038. if (kvm_load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS))
  4039. return 1;
  4040. if (kvm_load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS))
  4041. return 1;
  4042. if (kvm_load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS))
  4043. return 1;
  4044. return 0;
  4045. }
  4046. static int kvm_task_switch_16(struct kvm_vcpu *vcpu, u16 tss_selector,
  4047. u16 old_tss_sel, u32 old_tss_base,
  4048. struct desc_struct *nseg_desc)
  4049. {
  4050. struct tss_segment_16 tss_segment_16;
  4051. int ret = 0;
  4052. if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_16,
  4053. sizeof tss_segment_16))
  4054. goto out;
  4055. save_state_to_tss16(vcpu, &tss_segment_16);
  4056. if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_16,
  4057. sizeof tss_segment_16))
  4058. goto out;
  4059. if (kvm_read_guest(vcpu->kvm, get_tss_base_addr(vcpu, nseg_desc),
  4060. &tss_segment_16, sizeof tss_segment_16))
  4061. goto out;
  4062. if (old_tss_sel != 0xffff) {
  4063. tss_segment_16.prev_task_link = old_tss_sel;
  4064. if (kvm_write_guest(vcpu->kvm,
  4065. get_tss_base_addr(vcpu, nseg_desc),
  4066. &tss_segment_16.prev_task_link,
  4067. sizeof tss_segment_16.prev_task_link))
  4068. goto out;
  4069. }
  4070. if (load_state_from_tss16(vcpu, &tss_segment_16))
  4071. goto out;
  4072. ret = 1;
  4073. out:
  4074. return ret;
  4075. }
  4076. static int kvm_task_switch_32(struct kvm_vcpu *vcpu, u16 tss_selector,
  4077. u16 old_tss_sel, u32 old_tss_base,
  4078. struct desc_struct *nseg_desc)
  4079. {
  4080. struct tss_segment_32 tss_segment_32;
  4081. int ret = 0;
  4082. if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_32,
  4083. sizeof tss_segment_32))
  4084. goto out;
  4085. save_state_to_tss32(vcpu, &tss_segment_32);
  4086. if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_32,
  4087. sizeof tss_segment_32))
  4088. goto out;
  4089. if (kvm_read_guest(vcpu->kvm, get_tss_base_addr(vcpu, nseg_desc),
  4090. &tss_segment_32, sizeof tss_segment_32))
  4091. goto out;
  4092. if (old_tss_sel != 0xffff) {
  4093. tss_segment_32.prev_task_link = old_tss_sel;
  4094. if (kvm_write_guest(vcpu->kvm,
  4095. get_tss_base_addr(vcpu, nseg_desc),
  4096. &tss_segment_32.prev_task_link,
  4097. sizeof tss_segment_32.prev_task_link))
  4098. goto out;
  4099. }
  4100. if (load_state_from_tss32(vcpu, &tss_segment_32))
  4101. goto out;
  4102. ret = 1;
  4103. out:
  4104. return ret;
  4105. }
  4106. int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason)
  4107. {
  4108. struct kvm_segment tr_seg;
  4109. struct desc_struct cseg_desc;
  4110. struct desc_struct nseg_desc;
  4111. int ret = 0;
  4112. u32 old_tss_base = get_segment_base(vcpu, VCPU_SREG_TR);
  4113. u16 old_tss_sel = get_segment_selector(vcpu, VCPU_SREG_TR);
  4114. old_tss_base = vcpu->arch.mmu.gva_to_gpa(vcpu, old_tss_base);
  4115. /* FIXME: Handle errors. Failure to read either TSS or their
  4116. * descriptors should generate a pagefault.
  4117. */
  4118. if (load_guest_segment_descriptor(vcpu, tss_selector, &nseg_desc))
  4119. goto out;
  4120. if (load_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc))
  4121. goto out;
  4122. if (reason != TASK_SWITCH_IRET) {
  4123. int cpl;
  4124. cpl = kvm_x86_ops->get_cpl(vcpu);
  4125. if ((tss_selector & 3) > nseg_desc.dpl || cpl > nseg_desc.dpl) {
  4126. kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
  4127. return 1;
  4128. }
  4129. }
  4130. if (!nseg_desc.p || get_desc_limit(&nseg_desc) < 0x67) {
  4131. kvm_queue_exception_e(vcpu, TS_VECTOR, tss_selector & 0xfffc);
  4132. return 1;
  4133. }
  4134. if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
  4135. cseg_desc.type &= ~(1 << 1); //clear the B flag
  4136. save_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc);
  4137. }
  4138. if (reason == TASK_SWITCH_IRET) {
  4139. u32 eflags = kvm_get_rflags(vcpu);
  4140. kvm_set_rflags(vcpu, eflags & ~X86_EFLAGS_NT);
  4141. }
  4142. /* set back link to prev task only if NT bit is set in eflags
  4143. note that old_tss_sel is not used afetr this point */
  4144. if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
  4145. old_tss_sel = 0xffff;
  4146. if (nseg_desc.type & 8)
  4147. ret = kvm_task_switch_32(vcpu, tss_selector, old_tss_sel,
  4148. old_tss_base, &nseg_desc);
  4149. else
  4150. ret = kvm_task_switch_16(vcpu, tss_selector, old_tss_sel,
  4151. old_tss_base, &nseg_desc);
  4152. if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE) {
  4153. u32 eflags = kvm_get_rflags(vcpu);
  4154. kvm_set_rflags(vcpu, eflags | X86_EFLAGS_NT);
  4155. }
  4156. if (reason != TASK_SWITCH_IRET) {
  4157. nseg_desc.type |= (1 << 1);
  4158. save_guest_segment_descriptor(vcpu, tss_selector,
  4159. &nseg_desc);
  4160. }
  4161. kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 | X86_CR0_TS);
  4162. seg_desct_to_kvm_desct(&nseg_desc, tss_selector, &tr_seg);
  4163. tr_seg.type = 11;
  4164. kvm_set_segment(vcpu, &tr_seg, VCPU_SREG_TR);
  4165. out:
  4166. return ret;
  4167. }
  4168. EXPORT_SYMBOL_GPL(kvm_task_switch);
  4169. int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  4170. struct kvm_sregs *sregs)
  4171. {
  4172. int mmu_reset_needed = 0;
  4173. int pending_vec, max_bits;
  4174. struct descriptor_table dt;
  4175. vcpu_load(vcpu);
  4176. dt.limit = sregs->idt.limit;
  4177. dt.base = sregs->idt.base;
  4178. kvm_x86_ops->set_idt(vcpu, &dt);
  4179. dt.limit = sregs->gdt.limit;
  4180. dt.base = sregs->gdt.base;
  4181. kvm_x86_ops->set_gdt(vcpu, &dt);
  4182. vcpu->arch.cr2 = sregs->cr2;
  4183. mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
  4184. vcpu->arch.cr3 = sregs->cr3;
  4185. kvm_set_cr8(vcpu, sregs->cr8);
  4186. mmu_reset_needed |= vcpu->arch.shadow_efer != sregs->efer;
  4187. kvm_x86_ops->set_efer(vcpu, sregs->efer);
  4188. kvm_set_apic_base(vcpu, sregs->apic_base);
  4189. mmu_reset_needed |= vcpu->arch.cr0 != sregs->cr0;
  4190. kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
  4191. vcpu->arch.cr0 = sregs->cr0;
  4192. mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
  4193. kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
  4194. if (!is_long_mode(vcpu) && is_pae(vcpu)) {
  4195. load_pdptrs(vcpu, vcpu->arch.cr3);
  4196. mmu_reset_needed = 1;
  4197. }
  4198. if (mmu_reset_needed)
  4199. kvm_mmu_reset_context(vcpu);
  4200. max_bits = (sizeof sregs->interrupt_bitmap) << 3;
  4201. pending_vec = find_first_bit(
  4202. (const unsigned long *)sregs->interrupt_bitmap, max_bits);
  4203. if (pending_vec < max_bits) {
  4204. kvm_queue_interrupt(vcpu, pending_vec, false);
  4205. pr_debug("Set back pending irq %d\n", pending_vec);
  4206. if (irqchip_in_kernel(vcpu->kvm))
  4207. kvm_pic_clear_isr_ack(vcpu->kvm);
  4208. }
  4209. kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  4210. kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  4211. kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  4212. kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  4213. kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  4214. kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  4215. kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  4216. kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  4217. update_cr8_intercept(vcpu);
  4218. /* Older userspace won't unhalt the vcpu on reset. */
  4219. if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
  4220. sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
  4221. !(vcpu->arch.cr0 & X86_CR0_PE))
  4222. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  4223. vcpu_put(vcpu);
  4224. return 0;
  4225. }
  4226. int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
  4227. struct kvm_guest_debug *dbg)
  4228. {
  4229. unsigned long rflags;
  4230. int i, r;
  4231. vcpu_load(vcpu);
  4232. if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
  4233. r = -EBUSY;
  4234. if (vcpu->arch.exception.pending)
  4235. goto unlock_out;
  4236. if (dbg->control & KVM_GUESTDBG_INJECT_DB)
  4237. kvm_queue_exception(vcpu, DB_VECTOR);
  4238. else
  4239. kvm_queue_exception(vcpu, BP_VECTOR);
  4240. }
  4241. /*
  4242. * Read rflags as long as potentially injected trace flags are still
  4243. * filtered out.
  4244. */
  4245. rflags = kvm_get_rflags(vcpu);
  4246. vcpu->guest_debug = dbg->control;
  4247. if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
  4248. vcpu->guest_debug = 0;
  4249. if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
  4250. for (i = 0; i < KVM_NR_DB_REGS; ++i)
  4251. vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
  4252. vcpu->arch.switch_db_regs =
  4253. (dbg->arch.debugreg[7] & DR7_BP_EN_MASK);
  4254. } else {
  4255. for (i = 0; i < KVM_NR_DB_REGS; i++)
  4256. vcpu->arch.eff_db[i] = vcpu->arch.db[i];
  4257. vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK);
  4258. }
  4259. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
  4260. vcpu->arch.singlestep_cs =
  4261. get_segment_selector(vcpu, VCPU_SREG_CS);
  4262. vcpu->arch.singlestep_rip = kvm_rip_read(vcpu);
  4263. }
  4264. /*
  4265. * Trigger an rflags update that will inject or remove the trace
  4266. * flags.
  4267. */
  4268. kvm_set_rflags(vcpu, rflags);
  4269. kvm_x86_ops->set_guest_debug(vcpu, dbg);
  4270. r = 0;
  4271. unlock_out:
  4272. vcpu_put(vcpu);
  4273. return r;
  4274. }
  4275. /*
  4276. * fxsave fpu state. Taken from x86_64/processor.h. To be killed when
  4277. * we have asm/x86/processor.h
  4278. */
  4279. struct fxsave {
  4280. u16 cwd;
  4281. u16 swd;
  4282. u16 twd;
  4283. u16 fop;
  4284. u64 rip;
  4285. u64 rdp;
  4286. u32 mxcsr;
  4287. u32 mxcsr_mask;
  4288. u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */
  4289. #ifdef CONFIG_X86_64
  4290. u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */
  4291. #else
  4292. u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */
  4293. #endif
  4294. };
  4295. /*
  4296. * Translate a guest virtual address to a guest physical address.
  4297. */
  4298. int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  4299. struct kvm_translation *tr)
  4300. {
  4301. unsigned long vaddr = tr->linear_address;
  4302. gpa_t gpa;
  4303. int idx;
  4304. vcpu_load(vcpu);
  4305. idx = srcu_read_lock(&vcpu->kvm->srcu);
  4306. gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, vaddr);
  4307. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  4308. tr->physical_address = gpa;
  4309. tr->valid = gpa != UNMAPPED_GVA;
  4310. tr->writeable = 1;
  4311. tr->usermode = 0;
  4312. vcpu_put(vcpu);
  4313. return 0;
  4314. }
  4315. int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  4316. {
  4317. struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
  4318. vcpu_load(vcpu);
  4319. memcpy(fpu->fpr, fxsave->st_space, 128);
  4320. fpu->fcw = fxsave->cwd;
  4321. fpu->fsw = fxsave->swd;
  4322. fpu->ftwx = fxsave->twd;
  4323. fpu->last_opcode = fxsave->fop;
  4324. fpu->last_ip = fxsave->rip;
  4325. fpu->last_dp = fxsave->rdp;
  4326. memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
  4327. vcpu_put(vcpu);
  4328. return 0;
  4329. }
  4330. int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  4331. {
  4332. struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
  4333. vcpu_load(vcpu);
  4334. memcpy(fxsave->st_space, fpu->fpr, 128);
  4335. fxsave->cwd = fpu->fcw;
  4336. fxsave->swd = fpu->fsw;
  4337. fxsave->twd = fpu->ftwx;
  4338. fxsave->fop = fpu->last_opcode;
  4339. fxsave->rip = fpu->last_ip;
  4340. fxsave->rdp = fpu->last_dp;
  4341. memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
  4342. vcpu_put(vcpu);
  4343. return 0;
  4344. }
  4345. void fx_init(struct kvm_vcpu *vcpu)
  4346. {
  4347. unsigned after_mxcsr_mask;
  4348. /*
  4349. * Touch the fpu the first time in non atomic context as if
  4350. * this is the first fpu instruction the exception handler
  4351. * will fire before the instruction returns and it'll have to
  4352. * allocate ram with GFP_KERNEL.
  4353. */
  4354. if (!used_math())
  4355. kvm_fx_save(&vcpu->arch.host_fx_image);
  4356. /* Initialize guest FPU by resetting ours and saving into guest's */
  4357. preempt_disable();
  4358. kvm_fx_save(&vcpu->arch.host_fx_image);
  4359. kvm_fx_finit();
  4360. kvm_fx_save(&vcpu->arch.guest_fx_image);
  4361. kvm_fx_restore(&vcpu->arch.host_fx_image);
  4362. preempt_enable();
  4363. vcpu->arch.cr0 |= X86_CR0_ET;
  4364. after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
  4365. vcpu->arch.guest_fx_image.mxcsr = 0x1f80;
  4366. memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask,
  4367. 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
  4368. }
  4369. EXPORT_SYMBOL_GPL(fx_init);
  4370. void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
  4371. {
  4372. if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
  4373. return;
  4374. vcpu->guest_fpu_loaded = 1;
  4375. kvm_fx_save(&vcpu->arch.host_fx_image);
  4376. kvm_fx_restore(&vcpu->arch.guest_fx_image);
  4377. }
  4378. EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
  4379. void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
  4380. {
  4381. if (!vcpu->guest_fpu_loaded)
  4382. return;
  4383. vcpu->guest_fpu_loaded = 0;
  4384. kvm_fx_save(&vcpu->arch.guest_fx_image);
  4385. kvm_fx_restore(&vcpu->arch.host_fx_image);
  4386. ++vcpu->stat.fpu_reload;
  4387. }
  4388. EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
  4389. void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
  4390. {
  4391. if (vcpu->arch.time_page) {
  4392. kvm_release_page_dirty(vcpu->arch.time_page);
  4393. vcpu->arch.time_page = NULL;
  4394. }
  4395. kvm_x86_ops->vcpu_free(vcpu);
  4396. }
  4397. struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
  4398. unsigned int id)
  4399. {
  4400. return kvm_x86_ops->vcpu_create(kvm, id);
  4401. }
  4402. int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
  4403. {
  4404. int r;
  4405. /* We do fxsave: this must be aligned. */
  4406. BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF);
  4407. vcpu->arch.mtrr_state.have_fixed = 1;
  4408. vcpu_load(vcpu);
  4409. r = kvm_arch_vcpu_reset(vcpu);
  4410. if (r == 0)
  4411. r = kvm_mmu_setup(vcpu);
  4412. vcpu_put(vcpu);
  4413. if (r < 0)
  4414. goto free_vcpu;
  4415. return 0;
  4416. free_vcpu:
  4417. kvm_x86_ops->vcpu_free(vcpu);
  4418. return r;
  4419. }
  4420. void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
  4421. {
  4422. vcpu_load(vcpu);
  4423. kvm_mmu_unload(vcpu);
  4424. vcpu_put(vcpu);
  4425. kvm_x86_ops->vcpu_free(vcpu);
  4426. }
  4427. int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
  4428. {
  4429. vcpu->arch.nmi_pending = false;
  4430. vcpu->arch.nmi_injected = false;
  4431. vcpu->arch.switch_db_regs = 0;
  4432. memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
  4433. vcpu->arch.dr6 = DR6_FIXED_1;
  4434. vcpu->arch.dr7 = DR7_FIXED_1;
  4435. return kvm_x86_ops->vcpu_reset(vcpu);
  4436. }
  4437. int kvm_arch_hardware_enable(void *garbage)
  4438. {
  4439. /*
  4440. * Since this may be called from a hotplug notifcation,
  4441. * we can't get the CPU frequency directly.
  4442. */
  4443. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
  4444. int cpu = raw_smp_processor_id();
  4445. per_cpu(cpu_tsc_khz, cpu) = 0;
  4446. }
  4447. kvm_shared_msr_cpu_online();
  4448. return kvm_x86_ops->hardware_enable(garbage);
  4449. }
  4450. void kvm_arch_hardware_disable(void *garbage)
  4451. {
  4452. kvm_x86_ops->hardware_disable(garbage);
  4453. drop_user_return_notifiers(garbage);
  4454. }
  4455. int kvm_arch_hardware_setup(void)
  4456. {
  4457. return kvm_x86_ops->hardware_setup();
  4458. }
  4459. void kvm_arch_hardware_unsetup(void)
  4460. {
  4461. kvm_x86_ops->hardware_unsetup();
  4462. }
  4463. void kvm_arch_check_processor_compat(void *rtn)
  4464. {
  4465. kvm_x86_ops->check_processor_compatibility(rtn);
  4466. }
  4467. int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
  4468. {
  4469. struct page *page;
  4470. struct kvm *kvm;
  4471. int r;
  4472. BUG_ON(vcpu->kvm == NULL);
  4473. kvm = vcpu->kvm;
  4474. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  4475. if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
  4476. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  4477. else
  4478. vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
  4479. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  4480. if (!page) {
  4481. r = -ENOMEM;
  4482. goto fail;
  4483. }
  4484. vcpu->arch.pio_data = page_address(page);
  4485. r = kvm_mmu_create(vcpu);
  4486. if (r < 0)
  4487. goto fail_free_pio_data;
  4488. if (irqchip_in_kernel(kvm)) {
  4489. r = kvm_create_lapic(vcpu);
  4490. if (r < 0)
  4491. goto fail_mmu_destroy;
  4492. }
  4493. vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
  4494. GFP_KERNEL);
  4495. if (!vcpu->arch.mce_banks) {
  4496. r = -ENOMEM;
  4497. goto fail_free_lapic;
  4498. }
  4499. vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
  4500. return 0;
  4501. fail_free_lapic:
  4502. kvm_free_lapic(vcpu);
  4503. fail_mmu_destroy:
  4504. kvm_mmu_destroy(vcpu);
  4505. fail_free_pio_data:
  4506. free_page((unsigned long)vcpu->arch.pio_data);
  4507. fail:
  4508. return r;
  4509. }
  4510. void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
  4511. {
  4512. int idx;
  4513. kfree(vcpu->arch.mce_banks);
  4514. kvm_free_lapic(vcpu);
  4515. idx = srcu_read_lock(&vcpu->kvm->srcu);
  4516. kvm_mmu_destroy(vcpu);
  4517. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  4518. free_page((unsigned long)vcpu->arch.pio_data);
  4519. }
  4520. struct kvm *kvm_arch_create_vm(void)
  4521. {
  4522. struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
  4523. if (!kvm)
  4524. return ERR_PTR(-ENOMEM);
  4525. kvm->arch.aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
  4526. if (!kvm->arch.aliases) {
  4527. kfree(kvm);
  4528. return ERR_PTR(-ENOMEM);
  4529. }
  4530. INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
  4531. INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
  4532. /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
  4533. set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
  4534. rdtscll(kvm->arch.vm_init_tsc);
  4535. return kvm;
  4536. }
  4537. static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
  4538. {
  4539. vcpu_load(vcpu);
  4540. kvm_mmu_unload(vcpu);
  4541. vcpu_put(vcpu);
  4542. }
  4543. static void kvm_free_vcpus(struct kvm *kvm)
  4544. {
  4545. unsigned int i;
  4546. struct kvm_vcpu *vcpu;
  4547. /*
  4548. * Unpin any mmu pages first.
  4549. */
  4550. kvm_for_each_vcpu(i, vcpu, kvm)
  4551. kvm_unload_vcpu_mmu(vcpu);
  4552. kvm_for_each_vcpu(i, vcpu, kvm)
  4553. kvm_arch_vcpu_free(vcpu);
  4554. mutex_lock(&kvm->lock);
  4555. for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
  4556. kvm->vcpus[i] = NULL;
  4557. atomic_set(&kvm->online_vcpus, 0);
  4558. mutex_unlock(&kvm->lock);
  4559. }
  4560. void kvm_arch_sync_events(struct kvm *kvm)
  4561. {
  4562. kvm_free_all_assigned_devices(kvm);
  4563. }
  4564. void kvm_arch_destroy_vm(struct kvm *kvm)
  4565. {
  4566. kvm_iommu_unmap_guest(kvm);
  4567. kvm_free_pit(kvm);
  4568. kfree(kvm->arch.vpic);
  4569. kfree(kvm->arch.vioapic);
  4570. kvm_free_vcpus(kvm);
  4571. kvm_free_physmem(kvm);
  4572. if (kvm->arch.apic_access_page)
  4573. put_page(kvm->arch.apic_access_page);
  4574. if (kvm->arch.ept_identity_pagetable)
  4575. put_page(kvm->arch.ept_identity_pagetable);
  4576. kfree(kvm->arch.aliases);
  4577. kfree(kvm);
  4578. }
  4579. int kvm_arch_prepare_memory_region(struct kvm *kvm,
  4580. struct kvm_memory_slot *memslot,
  4581. struct kvm_memory_slot old,
  4582. struct kvm_userspace_memory_region *mem,
  4583. int user_alloc)
  4584. {
  4585. int npages = memslot->npages;
  4586. /*To keep backward compatibility with older userspace,
  4587. *x86 needs to hanlde !user_alloc case.
  4588. */
  4589. if (!user_alloc) {
  4590. if (npages && !old.rmap) {
  4591. unsigned long userspace_addr;
  4592. down_write(&current->mm->mmap_sem);
  4593. userspace_addr = do_mmap(NULL, 0,
  4594. npages * PAGE_SIZE,
  4595. PROT_READ | PROT_WRITE,
  4596. MAP_PRIVATE | MAP_ANONYMOUS,
  4597. 0);
  4598. up_write(&current->mm->mmap_sem);
  4599. if (IS_ERR((void *)userspace_addr))
  4600. return PTR_ERR((void *)userspace_addr);
  4601. memslot->userspace_addr = userspace_addr;
  4602. }
  4603. }
  4604. return 0;
  4605. }
  4606. void kvm_arch_commit_memory_region(struct kvm *kvm,
  4607. struct kvm_userspace_memory_region *mem,
  4608. struct kvm_memory_slot old,
  4609. int user_alloc)
  4610. {
  4611. int npages = mem->memory_size >> PAGE_SHIFT;
  4612. if (!user_alloc && !old.user_alloc && old.rmap && !npages) {
  4613. int ret;
  4614. down_write(&current->mm->mmap_sem);
  4615. ret = do_munmap(current->mm, old.userspace_addr,
  4616. old.npages * PAGE_SIZE);
  4617. up_write(&current->mm->mmap_sem);
  4618. if (ret < 0)
  4619. printk(KERN_WARNING
  4620. "kvm_vm_ioctl_set_memory_region: "
  4621. "failed to munmap memory\n");
  4622. }
  4623. spin_lock(&kvm->mmu_lock);
  4624. if (!kvm->arch.n_requested_mmu_pages) {
  4625. unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
  4626. kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
  4627. }
  4628. kvm_mmu_slot_remove_write_access(kvm, mem->slot);
  4629. spin_unlock(&kvm->mmu_lock);
  4630. }
  4631. void kvm_arch_flush_shadow(struct kvm *kvm)
  4632. {
  4633. kvm_mmu_zap_all(kvm);
  4634. kvm_reload_remote_mmus(kvm);
  4635. }
  4636. int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
  4637. {
  4638. return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
  4639. || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
  4640. || vcpu->arch.nmi_pending ||
  4641. (kvm_arch_interrupt_allowed(vcpu) &&
  4642. kvm_cpu_has_interrupt(vcpu));
  4643. }
  4644. void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
  4645. {
  4646. int me;
  4647. int cpu = vcpu->cpu;
  4648. if (waitqueue_active(&vcpu->wq)) {
  4649. wake_up_interruptible(&vcpu->wq);
  4650. ++vcpu->stat.halt_wakeup;
  4651. }
  4652. me = get_cpu();
  4653. if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
  4654. if (!test_and_set_bit(KVM_REQ_KICK, &vcpu->requests))
  4655. smp_send_reschedule(cpu);
  4656. put_cpu();
  4657. }
  4658. int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
  4659. {
  4660. return kvm_x86_ops->interrupt_allowed(vcpu);
  4661. }
  4662. unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
  4663. {
  4664. unsigned long rflags;
  4665. rflags = kvm_x86_ops->get_rflags(vcpu);
  4666. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
  4667. rflags &= ~(unsigned long)(X86_EFLAGS_TF | X86_EFLAGS_RF);
  4668. return rflags;
  4669. }
  4670. EXPORT_SYMBOL_GPL(kvm_get_rflags);
  4671. void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  4672. {
  4673. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
  4674. vcpu->arch.singlestep_cs ==
  4675. get_segment_selector(vcpu, VCPU_SREG_CS) &&
  4676. vcpu->arch.singlestep_rip == kvm_rip_read(vcpu))
  4677. rflags |= X86_EFLAGS_TF | X86_EFLAGS_RF;
  4678. kvm_x86_ops->set_rflags(vcpu, rflags);
  4679. }
  4680. EXPORT_SYMBOL_GPL(kvm_set_rflags);
  4681. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
  4682. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
  4683. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
  4684. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
  4685. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
  4686. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
  4687. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
  4688. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
  4689. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
  4690. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
  4691. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);