x86.c 169 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * derived from drivers/kvm/kvm_main.c
  5. *
  6. * Copyright (C) 2006 Qumranet, Inc.
  7. * Copyright (C) 2008 Qumranet, Inc.
  8. * Copyright IBM Corporation, 2008
  9. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  10. *
  11. * Authors:
  12. * Avi Kivity <avi@qumranet.com>
  13. * Yaniv Kamay <yaniv@qumranet.com>
  14. * Amit Shah <amit.shah@qumranet.com>
  15. * Ben-Ami Yassour <benami@il.ibm.com>
  16. *
  17. * This work is licensed under the terms of the GNU GPL, version 2. See
  18. * the COPYING file in the top-level directory.
  19. *
  20. */
  21. #include <linux/kvm_host.h>
  22. #include "irq.h"
  23. #include "mmu.h"
  24. #include "i8254.h"
  25. #include "tss.h"
  26. #include "kvm_cache_regs.h"
  27. #include "x86.h"
  28. #include <linux/clocksource.h>
  29. #include <linux/interrupt.h>
  30. #include <linux/kvm.h>
  31. #include <linux/fs.h>
  32. #include <linux/vmalloc.h>
  33. #include <linux/module.h>
  34. #include <linux/mman.h>
  35. #include <linux/highmem.h>
  36. #include <linux/iommu.h>
  37. #include <linux/intel-iommu.h>
  38. #include <linux/cpufreq.h>
  39. #include <linux/user-return-notifier.h>
  40. #include <linux/srcu.h>
  41. #include <linux/slab.h>
  42. #include <linux/perf_event.h>
  43. #include <linux/uaccess.h>
  44. #include <linux/hash.h>
  45. #include <trace/events/kvm.h>
  46. #define CREATE_TRACE_POINTS
  47. #include "trace.h"
  48. #include <asm/debugreg.h>
  49. #include <asm/msr.h>
  50. #include <asm/desc.h>
  51. #include <asm/mtrr.h>
  52. #include <asm/mce.h>
  53. #include <asm/i387.h>
  54. #include <asm/xcr.h>
  55. #include <asm/pvclock.h>
  56. #include <asm/div64.h>
  57. #define MAX_IO_MSRS 256
  58. #define KVM_MAX_MCE_BANKS 32
  59. #define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P)
  60. #define emul_to_vcpu(ctxt) \
  61. container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)
  62. /* EFER defaults:
  63. * - enable syscall per default because its emulated by KVM
  64. * - enable LME and LMA per default on 64 bit KVM
  65. */
  66. #ifdef CONFIG_X86_64
  67. static
  68. u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
  69. #else
  70. static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
  71. #endif
  72. #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  73. #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  74. static void update_cr8_intercept(struct kvm_vcpu *vcpu);
  75. static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  76. struct kvm_cpuid_entry2 __user *entries);
  77. struct kvm_x86_ops *kvm_x86_ops;
  78. EXPORT_SYMBOL_GPL(kvm_x86_ops);
  79. int ignore_msrs = 0;
  80. module_param_named(ignore_msrs, ignore_msrs, bool, S_IRUGO | S_IWUSR);
  81. bool kvm_has_tsc_control;
  82. EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
  83. u32 kvm_max_guest_tsc_khz;
  84. EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
  85. #define KVM_NR_SHARED_MSRS 16
  86. struct kvm_shared_msrs_global {
  87. int nr;
  88. u32 msrs[KVM_NR_SHARED_MSRS];
  89. };
  90. struct kvm_shared_msrs {
  91. struct user_return_notifier urn;
  92. bool registered;
  93. struct kvm_shared_msr_values {
  94. u64 host;
  95. u64 curr;
  96. } values[KVM_NR_SHARED_MSRS];
  97. };
  98. static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
  99. static DEFINE_PER_CPU(struct kvm_shared_msrs, shared_msrs);
  100. struct kvm_stats_debugfs_item debugfs_entries[] = {
  101. { "pf_fixed", VCPU_STAT(pf_fixed) },
  102. { "pf_guest", VCPU_STAT(pf_guest) },
  103. { "tlb_flush", VCPU_STAT(tlb_flush) },
  104. { "invlpg", VCPU_STAT(invlpg) },
  105. { "exits", VCPU_STAT(exits) },
  106. { "io_exits", VCPU_STAT(io_exits) },
  107. { "mmio_exits", VCPU_STAT(mmio_exits) },
  108. { "signal_exits", VCPU_STAT(signal_exits) },
  109. { "irq_window", VCPU_STAT(irq_window_exits) },
  110. { "nmi_window", VCPU_STAT(nmi_window_exits) },
  111. { "halt_exits", VCPU_STAT(halt_exits) },
  112. { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  113. { "hypercalls", VCPU_STAT(hypercalls) },
  114. { "request_irq", VCPU_STAT(request_irq_exits) },
  115. { "irq_exits", VCPU_STAT(irq_exits) },
  116. { "host_state_reload", VCPU_STAT(host_state_reload) },
  117. { "efer_reload", VCPU_STAT(efer_reload) },
  118. { "fpu_reload", VCPU_STAT(fpu_reload) },
  119. { "insn_emulation", VCPU_STAT(insn_emulation) },
  120. { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
  121. { "irq_injections", VCPU_STAT(irq_injections) },
  122. { "nmi_injections", VCPU_STAT(nmi_injections) },
  123. { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
  124. { "mmu_pte_write", VM_STAT(mmu_pte_write) },
  125. { "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
  126. { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
  127. { "mmu_flooded", VM_STAT(mmu_flooded) },
  128. { "mmu_recycled", VM_STAT(mmu_recycled) },
  129. { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
  130. { "mmu_unsync", VM_STAT(mmu_unsync) },
  131. { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
  132. { "largepages", VM_STAT(lpages) },
  133. { NULL }
  134. };
  135. u64 __read_mostly host_xcr0;
  136. int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
  137. static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
  138. {
  139. int i;
  140. for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
  141. vcpu->arch.apf.gfns[i] = ~0;
  142. }
  143. static void kvm_on_user_return(struct user_return_notifier *urn)
  144. {
  145. unsigned slot;
  146. struct kvm_shared_msrs *locals
  147. = container_of(urn, struct kvm_shared_msrs, urn);
  148. struct kvm_shared_msr_values *values;
  149. for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
  150. values = &locals->values[slot];
  151. if (values->host != values->curr) {
  152. wrmsrl(shared_msrs_global.msrs[slot], values->host);
  153. values->curr = values->host;
  154. }
  155. }
  156. locals->registered = false;
  157. user_return_notifier_unregister(urn);
  158. }
  159. static void shared_msr_update(unsigned slot, u32 msr)
  160. {
  161. struct kvm_shared_msrs *smsr;
  162. u64 value;
  163. smsr = &__get_cpu_var(shared_msrs);
  164. /* only read, and nobody should modify it at this time,
  165. * so don't need lock */
  166. if (slot >= shared_msrs_global.nr) {
  167. printk(KERN_ERR "kvm: invalid MSR slot!");
  168. return;
  169. }
  170. rdmsrl_safe(msr, &value);
  171. smsr->values[slot].host = value;
  172. smsr->values[slot].curr = value;
  173. }
  174. void kvm_define_shared_msr(unsigned slot, u32 msr)
  175. {
  176. if (slot >= shared_msrs_global.nr)
  177. shared_msrs_global.nr = slot + 1;
  178. shared_msrs_global.msrs[slot] = msr;
  179. /* we need ensured the shared_msr_global have been updated */
  180. smp_wmb();
  181. }
  182. EXPORT_SYMBOL_GPL(kvm_define_shared_msr);
  183. static void kvm_shared_msr_cpu_online(void)
  184. {
  185. unsigned i;
  186. for (i = 0; i < shared_msrs_global.nr; ++i)
  187. shared_msr_update(i, shared_msrs_global.msrs[i]);
  188. }
  189. void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
  190. {
  191. struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);
  192. if (((value ^ smsr->values[slot].curr) & mask) == 0)
  193. return;
  194. smsr->values[slot].curr = value;
  195. wrmsrl(shared_msrs_global.msrs[slot], value);
  196. if (!smsr->registered) {
  197. smsr->urn.on_user_return = kvm_on_user_return;
  198. user_return_notifier_register(&smsr->urn);
  199. smsr->registered = true;
  200. }
  201. }
  202. EXPORT_SYMBOL_GPL(kvm_set_shared_msr);
  203. static void drop_user_return_notifiers(void *ignore)
  204. {
  205. struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);
  206. if (smsr->registered)
  207. kvm_on_user_return(&smsr->urn);
  208. }
  209. u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
  210. {
  211. if (irqchip_in_kernel(vcpu->kvm))
  212. return vcpu->arch.apic_base;
  213. else
  214. return vcpu->arch.apic_base;
  215. }
  216. EXPORT_SYMBOL_GPL(kvm_get_apic_base);
  217. void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
  218. {
  219. /* TODO: reserve bits check */
  220. if (irqchip_in_kernel(vcpu->kvm))
  221. kvm_lapic_set_base(vcpu, data);
  222. else
  223. vcpu->arch.apic_base = data;
  224. }
  225. EXPORT_SYMBOL_GPL(kvm_set_apic_base);
  226. #define EXCPT_BENIGN 0
  227. #define EXCPT_CONTRIBUTORY 1
  228. #define EXCPT_PF 2
  229. static int exception_class(int vector)
  230. {
  231. switch (vector) {
  232. case PF_VECTOR:
  233. return EXCPT_PF;
  234. case DE_VECTOR:
  235. case TS_VECTOR:
  236. case NP_VECTOR:
  237. case SS_VECTOR:
  238. case GP_VECTOR:
  239. return EXCPT_CONTRIBUTORY;
  240. default:
  241. break;
  242. }
  243. return EXCPT_BENIGN;
  244. }
  245. static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
  246. unsigned nr, bool has_error, u32 error_code,
  247. bool reinject)
  248. {
  249. u32 prev_nr;
  250. int class1, class2;
  251. kvm_make_request(KVM_REQ_EVENT, vcpu);
  252. if (!vcpu->arch.exception.pending) {
  253. queue:
  254. vcpu->arch.exception.pending = true;
  255. vcpu->arch.exception.has_error_code = has_error;
  256. vcpu->arch.exception.nr = nr;
  257. vcpu->arch.exception.error_code = error_code;
  258. vcpu->arch.exception.reinject = reinject;
  259. return;
  260. }
  261. /* to check exception */
  262. prev_nr = vcpu->arch.exception.nr;
  263. if (prev_nr == DF_VECTOR) {
  264. /* triple fault -> shutdown */
  265. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  266. return;
  267. }
  268. class1 = exception_class(prev_nr);
  269. class2 = exception_class(nr);
  270. if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
  271. || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
  272. /* generate double fault per SDM Table 5-5 */
  273. vcpu->arch.exception.pending = true;
  274. vcpu->arch.exception.has_error_code = true;
  275. vcpu->arch.exception.nr = DF_VECTOR;
  276. vcpu->arch.exception.error_code = 0;
  277. } else
  278. /* replace previous exception with a new one in a hope
  279. that instruction re-execution will regenerate lost
  280. exception */
  281. goto queue;
  282. }
  283. void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
  284. {
  285. kvm_multiple_exception(vcpu, nr, false, 0, false);
  286. }
  287. EXPORT_SYMBOL_GPL(kvm_queue_exception);
  288. void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
  289. {
  290. kvm_multiple_exception(vcpu, nr, false, 0, true);
  291. }
  292. EXPORT_SYMBOL_GPL(kvm_requeue_exception);
  293. void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
  294. {
  295. if (err)
  296. kvm_inject_gp(vcpu, 0);
  297. else
  298. kvm_x86_ops->skip_emulated_instruction(vcpu);
  299. }
  300. EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
  301. void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
  302. {
  303. ++vcpu->stat.pf_guest;
  304. vcpu->arch.cr2 = fault->address;
  305. kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
  306. }
  307. EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
  308. void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
  309. {
  310. if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
  311. vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
  312. else
  313. vcpu->arch.mmu.inject_page_fault(vcpu, fault);
  314. }
  315. void kvm_inject_nmi(struct kvm_vcpu *vcpu)
  316. {
  317. kvm_make_request(KVM_REQ_EVENT, vcpu);
  318. vcpu->arch.nmi_pending = 1;
  319. }
  320. EXPORT_SYMBOL_GPL(kvm_inject_nmi);
  321. void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
  322. {
  323. kvm_multiple_exception(vcpu, nr, true, error_code, false);
  324. }
  325. EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
  326. void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
  327. {
  328. kvm_multiple_exception(vcpu, nr, true, error_code, true);
  329. }
  330. EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);
  331. /*
  332. * Checks if cpl <= required_cpl; if true, return true. Otherwise queue
  333. * a #GP and return false.
  334. */
  335. bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
  336. {
  337. if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
  338. return true;
  339. kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
  340. return false;
  341. }
  342. EXPORT_SYMBOL_GPL(kvm_require_cpl);
  343. /*
  344. * This function will be used to read from the physical memory of the currently
  345. * running guest. The difference to kvm_read_guest_page is that this function
  346. * can read from guest physical or from the guest's guest physical memory.
  347. */
  348. int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
  349. gfn_t ngfn, void *data, int offset, int len,
  350. u32 access)
  351. {
  352. gfn_t real_gfn;
  353. gpa_t ngpa;
  354. ngpa = gfn_to_gpa(ngfn);
  355. real_gfn = mmu->translate_gpa(vcpu, ngpa, access);
  356. if (real_gfn == UNMAPPED_GVA)
  357. return -EFAULT;
  358. real_gfn = gpa_to_gfn(real_gfn);
  359. return kvm_read_guest_page(vcpu->kvm, real_gfn, data, offset, len);
  360. }
  361. EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);
  362. int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
  363. void *data, int offset, int len, u32 access)
  364. {
  365. return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
  366. data, offset, len, access);
  367. }
  368. /*
  369. * Load the pae pdptrs. Return true is they are all valid.
  370. */
  371. int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
  372. {
  373. gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
  374. unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
  375. int i;
  376. int ret;
  377. u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
  378. ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
  379. offset * sizeof(u64), sizeof(pdpte),
  380. PFERR_USER_MASK|PFERR_WRITE_MASK);
  381. if (ret < 0) {
  382. ret = 0;
  383. goto out;
  384. }
  385. for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
  386. if (is_present_gpte(pdpte[i]) &&
  387. (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
  388. ret = 0;
  389. goto out;
  390. }
  391. }
  392. ret = 1;
  393. memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
  394. __set_bit(VCPU_EXREG_PDPTR,
  395. (unsigned long *)&vcpu->arch.regs_avail);
  396. __set_bit(VCPU_EXREG_PDPTR,
  397. (unsigned long *)&vcpu->arch.regs_dirty);
  398. out:
  399. return ret;
  400. }
  401. EXPORT_SYMBOL_GPL(load_pdptrs);
  402. static bool pdptrs_changed(struct kvm_vcpu *vcpu)
  403. {
  404. u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
  405. bool changed = true;
  406. int offset;
  407. gfn_t gfn;
  408. int r;
  409. if (is_long_mode(vcpu) || !is_pae(vcpu))
  410. return false;
  411. if (!test_bit(VCPU_EXREG_PDPTR,
  412. (unsigned long *)&vcpu->arch.regs_avail))
  413. return true;
  414. gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT;
  415. offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1);
  416. r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
  417. PFERR_USER_MASK | PFERR_WRITE_MASK);
  418. if (r < 0)
  419. goto out;
  420. changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
  421. out:
  422. return changed;
  423. }
  424. int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  425. {
  426. unsigned long old_cr0 = kvm_read_cr0(vcpu);
  427. unsigned long update_bits = X86_CR0_PG | X86_CR0_WP |
  428. X86_CR0_CD | X86_CR0_NW;
  429. cr0 |= X86_CR0_ET;
  430. #ifdef CONFIG_X86_64
  431. if (cr0 & 0xffffffff00000000UL)
  432. return 1;
  433. #endif
  434. cr0 &= ~CR0_RESERVED_BITS;
  435. if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
  436. return 1;
  437. if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
  438. return 1;
  439. if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
  440. #ifdef CONFIG_X86_64
  441. if ((vcpu->arch.efer & EFER_LME)) {
  442. int cs_db, cs_l;
  443. if (!is_pae(vcpu))
  444. return 1;
  445. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  446. if (cs_l)
  447. return 1;
  448. } else
  449. #endif
  450. if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
  451. kvm_read_cr3(vcpu)))
  452. return 1;
  453. }
  454. kvm_x86_ops->set_cr0(vcpu, cr0);
  455. if ((cr0 ^ old_cr0) & X86_CR0_PG) {
  456. kvm_clear_async_pf_completion_queue(vcpu);
  457. kvm_async_pf_hash_reset(vcpu);
  458. }
  459. if ((cr0 ^ old_cr0) & update_bits)
  460. kvm_mmu_reset_context(vcpu);
  461. return 0;
  462. }
  463. EXPORT_SYMBOL_GPL(kvm_set_cr0);
  464. void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
  465. {
  466. (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
  467. }
  468. EXPORT_SYMBOL_GPL(kvm_lmsw);
  469. int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
  470. {
  471. u64 xcr0;
  472. /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now */
  473. if (index != XCR_XFEATURE_ENABLED_MASK)
  474. return 1;
  475. xcr0 = xcr;
  476. if (kvm_x86_ops->get_cpl(vcpu) != 0)
  477. return 1;
  478. if (!(xcr0 & XSTATE_FP))
  479. return 1;
  480. if ((xcr0 & XSTATE_YMM) && !(xcr0 & XSTATE_SSE))
  481. return 1;
  482. if (xcr0 & ~host_xcr0)
  483. return 1;
  484. vcpu->arch.xcr0 = xcr0;
  485. vcpu->guest_xcr0_loaded = 0;
  486. return 0;
  487. }
  488. int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
  489. {
  490. if (__kvm_set_xcr(vcpu, index, xcr)) {
  491. kvm_inject_gp(vcpu, 0);
  492. return 1;
  493. }
  494. return 0;
  495. }
  496. EXPORT_SYMBOL_GPL(kvm_set_xcr);
  497. static bool guest_cpuid_has_xsave(struct kvm_vcpu *vcpu)
  498. {
  499. struct kvm_cpuid_entry2 *best;
  500. best = kvm_find_cpuid_entry(vcpu, 1, 0);
  501. return best && (best->ecx & bit(X86_FEATURE_XSAVE));
  502. }
  503. static bool guest_cpuid_has_smep(struct kvm_vcpu *vcpu)
  504. {
  505. struct kvm_cpuid_entry2 *best;
  506. best = kvm_find_cpuid_entry(vcpu, 7, 0);
  507. return best && (best->ebx & bit(X86_FEATURE_SMEP));
  508. }
  509. static bool guest_cpuid_has_fsgsbase(struct kvm_vcpu *vcpu)
  510. {
  511. struct kvm_cpuid_entry2 *best;
  512. best = kvm_find_cpuid_entry(vcpu, 7, 0);
  513. return best && (best->ebx & bit(X86_FEATURE_FSGSBASE));
  514. }
  515. static void update_cpuid(struct kvm_vcpu *vcpu)
  516. {
  517. struct kvm_cpuid_entry2 *best;
  518. best = kvm_find_cpuid_entry(vcpu, 1, 0);
  519. if (!best)
  520. return;
  521. /* Update OSXSAVE bit */
  522. if (cpu_has_xsave && best->function == 0x1) {
  523. best->ecx &= ~(bit(X86_FEATURE_OSXSAVE));
  524. if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE))
  525. best->ecx |= bit(X86_FEATURE_OSXSAVE);
  526. }
  527. }
  528. int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  529. {
  530. unsigned long old_cr4 = kvm_read_cr4(vcpu);
  531. unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE |
  532. X86_CR4_PAE | X86_CR4_SMEP;
  533. if (cr4 & CR4_RESERVED_BITS)
  534. return 1;
  535. if (!guest_cpuid_has_xsave(vcpu) && (cr4 & X86_CR4_OSXSAVE))
  536. return 1;
  537. if (!guest_cpuid_has_smep(vcpu) && (cr4 & X86_CR4_SMEP))
  538. return 1;
  539. if (!guest_cpuid_has_fsgsbase(vcpu) && (cr4 & X86_CR4_RDWRGSFS))
  540. return 1;
  541. if (is_long_mode(vcpu)) {
  542. if (!(cr4 & X86_CR4_PAE))
  543. return 1;
  544. } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
  545. && ((cr4 ^ old_cr4) & pdptr_bits)
  546. && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
  547. kvm_read_cr3(vcpu)))
  548. return 1;
  549. if (kvm_x86_ops->set_cr4(vcpu, cr4))
  550. return 1;
  551. if ((cr4 ^ old_cr4) & pdptr_bits)
  552. kvm_mmu_reset_context(vcpu);
  553. if ((cr4 ^ old_cr4) & X86_CR4_OSXSAVE)
  554. update_cpuid(vcpu);
  555. return 0;
  556. }
  557. EXPORT_SYMBOL_GPL(kvm_set_cr4);
  558. int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
  559. {
  560. if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
  561. kvm_mmu_sync_roots(vcpu);
  562. kvm_mmu_flush_tlb(vcpu);
  563. return 0;
  564. }
  565. if (is_long_mode(vcpu)) {
  566. if (cr3 & CR3_L_MODE_RESERVED_BITS)
  567. return 1;
  568. } else {
  569. if (is_pae(vcpu)) {
  570. if (cr3 & CR3_PAE_RESERVED_BITS)
  571. return 1;
  572. if (is_paging(vcpu) &&
  573. !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
  574. return 1;
  575. }
  576. /*
  577. * We don't check reserved bits in nonpae mode, because
  578. * this isn't enforced, and VMware depends on this.
  579. */
  580. }
  581. /*
  582. * Does the new cr3 value map to physical memory? (Note, we
  583. * catch an invalid cr3 even in real-mode, because it would
  584. * cause trouble later on when we turn on paging anyway.)
  585. *
  586. * A real CPU would silently accept an invalid cr3 and would
  587. * attempt to use it - with largely undefined (and often hard
  588. * to debug) behavior on the guest side.
  589. */
  590. if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
  591. return 1;
  592. vcpu->arch.cr3 = cr3;
  593. __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
  594. vcpu->arch.mmu.new_cr3(vcpu);
  595. return 0;
  596. }
  597. EXPORT_SYMBOL_GPL(kvm_set_cr3);
  598. int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
  599. {
  600. if (cr8 & CR8_RESERVED_BITS)
  601. return 1;
  602. if (irqchip_in_kernel(vcpu->kvm))
  603. kvm_lapic_set_tpr(vcpu, cr8);
  604. else
  605. vcpu->arch.cr8 = cr8;
  606. return 0;
  607. }
  608. EXPORT_SYMBOL_GPL(kvm_set_cr8);
  609. unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
  610. {
  611. if (irqchip_in_kernel(vcpu->kvm))
  612. return kvm_lapic_get_cr8(vcpu);
  613. else
  614. return vcpu->arch.cr8;
  615. }
  616. EXPORT_SYMBOL_GPL(kvm_get_cr8);
  617. static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
  618. {
  619. switch (dr) {
  620. case 0 ... 3:
  621. vcpu->arch.db[dr] = val;
  622. if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
  623. vcpu->arch.eff_db[dr] = val;
  624. break;
  625. case 4:
  626. if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
  627. return 1; /* #UD */
  628. /* fall through */
  629. case 6:
  630. if (val & 0xffffffff00000000ULL)
  631. return -1; /* #GP */
  632. vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
  633. break;
  634. case 5:
  635. if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
  636. return 1; /* #UD */
  637. /* fall through */
  638. default: /* 7 */
  639. if (val & 0xffffffff00000000ULL)
  640. return -1; /* #GP */
  641. vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
  642. if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
  643. kvm_x86_ops->set_dr7(vcpu, vcpu->arch.dr7);
  644. vcpu->arch.switch_db_regs = (val & DR7_BP_EN_MASK);
  645. }
  646. break;
  647. }
  648. return 0;
  649. }
  650. int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
  651. {
  652. int res;
  653. res = __kvm_set_dr(vcpu, dr, val);
  654. if (res > 0)
  655. kvm_queue_exception(vcpu, UD_VECTOR);
  656. else if (res < 0)
  657. kvm_inject_gp(vcpu, 0);
  658. return res;
  659. }
  660. EXPORT_SYMBOL_GPL(kvm_set_dr);
  661. static int _kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
  662. {
  663. switch (dr) {
  664. case 0 ... 3:
  665. *val = vcpu->arch.db[dr];
  666. break;
  667. case 4:
  668. if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
  669. return 1;
  670. /* fall through */
  671. case 6:
  672. *val = vcpu->arch.dr6;
  673. break;
  674. case 5:
  675. if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
  676. return 1;
  677. /* fall through */
  678. default: /* 7 */
  679. *val = vcpu->arch.dr7;
  680. break;
  681. }
  682. return 0;
  683. }
  684. int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
  685. {
  686. if (_kvm_get_dr(vcpu, dr, val)) {
  687. kvm_queue_exception(vcpu, UD_VECTOR);
  688. return 1;
  689. }
  690. return 0;
  691. }
  692. EXPORT_SYMBOL_GPL(kvm_get_dr);
  693. /*
  694. * List of msr numbers which we expose to userspace through KVM_GET_MSRS
  695. * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
  696. *
  697. * This list is modified at module load time to reflect the
  698. * capabilities of the host cpu. This capabilities test skips MSRs that are
  699. * kvm-specific. Those are put in the beginning of the list.
  700. */
  701. #define KVM_SAVE_MSRS_BEGIN 9
  702. static u32 msrs_to_save[] = {
  703. MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
  704. MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
  705. HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
  706. HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
  707. MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  708. MSR_STAR,
  709. #ifdef CONFIG_X86_64
  710. MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
  711. #endif
  712. MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA
  713. };
  714. static unsigned num_msrs_to_save;
  715. static u32 emulated_msrs[] = {
  716. MSR_IA32_MISC_ENABLE,
  717. MSR_IA32_MCG_STATUS,
  718. MSR_IA32_MCG_CTL,
  719. };
  720. static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
  721. {
  722. u64 old_efer = vcpu->arch.efer;
  723. if (efer & efer_reserved_bits)
  724. return 1;
  725. if (is_paging(vcpu)
  726. && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
  727. return 1;
  728. if (efer & EFER_FFXSR) {
  729. struct kvm_cpuid_entry2 *feat;
  730. feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
  731. if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT)))
  732. return 1;
  733. }
  734. if (efer & EFER_SVME) {
  735. struct kvm_cpuid_entry2 *feat;
  736. feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
  737. if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM)))
  738. return 1;
  739. }
  740. efer &= ~EFER_LMA;
  741. efer |= vcpu->arch.efer & EFER_LMA;
  742. kvm_x86_ops->set_efer(vcpu, efer);
  743. vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
  744. /* Update reserved bits */
  745. if ((efer ^ old_efer) & EFER_NX)
  746. kvm_mmu_reset_context(vcpu);
  747. return 0;
  748. }
  749. void kvm_enable_efer_bits(u64 mask)
  750. {
  751. efer_reserved_bits &= ~mask;
  752. }
  753. EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
  754. /*
  755. * Writes msr value into into the appropriate "register".
  756. * Returns 0 on success, non-0 otherwise.
  757. * Assumes vcpu_load() was already called.
  758. */
  759. int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
  760. {
  761. return kvm_x86_ops->set_msr(vcpu, msr_index, data);
  762. }
  763. /*
  764. * Adapt set_msr() to msr_io()'s calling convention
  765. */
  766. static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
  767. {
  768. return kvm_set_msr(vcpu, index, *data);
  769. }
  770. static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
  771. {
  772. int version;
  773. int r;
  774. struct pvclock_wall_clock wc;
  775. struct timespec boot;
  776. if (!wall_clock)
  777. return;
  778. r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
  779. if (r)
  780. return;
  781. if (version & 1)
  782. ++version; /* first time write, random junk */
  783. ++version;
  784. kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
  785. /*
  786. * The guest calculates current wall clock time by adding
  787. * system time (updated by kvm_guest_time_update below) to the
  788. * wall clock specified here. guest system time equals host
  789. * system time for us, thus we must fill in host boot time here.
  790. */
  791. getboottime(&boot);
  792. wc.sec = boot.tv_sec;
  793. wc.nsec = boot.tv_nsec;
  794. wc.version = version;
  795. kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
  796. version++;
  797. kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
  798. }
  799. static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
  800. {
  801. uint32_t quotient, remainder;
  802. /* Don't try to replace with do_div(), this one calculates
  803. * "(dividend << 32) / divisor" */
  804. __asm__ ( "divl %4"
  805. : "=a" (quotient), "=d" (remainder)
  806. : "0" (0), "1" (dividend), "r" (divisor) );
  807. return quotient;
  808. }
  809. static void kvm_get_time_scale(uint32_t scaled_khz, uint32_t base_khz,
  810. s8 *pshift, u32 *pmultiplier)
  811. {
  812. uint64_t scaled64;
  813. int32_t shift = 0;
  814. uint64_t tps64;
  815. uint32_t tps32;
  816. tps64 = base_khz * 1000LL;
  817. scaled64 = scaled_khz * 1000LL;
  818. while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
  819. tps64 >>= 1;
  820. shift--;
  821. }
  822. tps32 = (uint32_t)tps64;
  823. while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
  824. if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
  825. scaled64 >>= 1;
  826. else
  827. tps32 <<= 1;
  828. shift++;
  829. }
  830. *pshift = shift;
  831. *pmultiplier = div_frac(scaled64, tps32);
  832. pr_debug("%s: base_khz %u => %u, shift %d, mul %u\n",
  833. __func__, base_khz, scaled_khz, shift, *pmultiplier);
  834. }
  835. static inline u64 get_kernel_ns(void)
  836. {
  837. struct timespec ts;
  838. WARN_ON(preemptible());
  839. ktime_get_ts(&ts);
  840. monotonic_to_bootbased(&ts);
  841. return timespec_to_ns(&ts);
  842. }
  843. static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
  844. unsigned long max_tsc_khz;
  845. static inline int kvm_tsc_changes_freq(void)
  846. {
  847. int cpu = get_cpu();
  848. int ret = !boot_cpu_has(X86_FEATURE_CONSTANT_TSC) &&
  849. cpufreq_quick_get(cpu) != 0;
  850. put_cpu();
  851. return ret;
  852. }
  853. static u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu)
  854. {
  855. if (vcpu->arch.virtual_tsc_khz)
  856. return vcpu->arch.virtual_tsc_khz;
  857. else
  858. return __this_cpu_read(cpu_tsc_khz);
  859. }
  860. static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
  861. {
  862. u64 ret;
  863. WARN_ON(preemptible());
  864. if (kvm_tsc_changes_freq())
  865. printk_once(KERN_WARNING
  866. "kvm: unreliable cycle conversion on adjustable rate TSC\n");
  867. ret = nsec * vcpu_tsc_khz(vcpu);
  868. do_div(ret, USEC_PER_SEC);
  869. return ret;
  870. }
  871. static void kvm_init_tsc_catchup(struct kvm_vcpu *vcpu, u32 this_tsc_khz)
  872. {
  873. /* Compute a scale to convert nanoseconds in TSC cycles */
  874. kvm_get_time_scale(this_tsc_khz, NSEC_PER_SEC / 1000,
  875. &vcpu->arch.tsc_catchup_shift,
  876. &vcpu->arch.tsc_catchup_mult);
  877. }
  878. static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
  879. {
  880. u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.last_tsc_nsec,
  881. vcpu->arch.tsc_catchup_mult,
  882. vcpu->arch.tsc_catchup_shift);
  883. tsc += vcpu->arch.last_tsc_write;
  884. return tsc;
  885. }
  886. void kvm_write_tsc(struct kvm_vcpu *vcpu, u64 data)
  887. {
  888. struct kvm *kvm = vcpu->kvm;
  889. u64 offset, ns, elapsed;
  890. unsigned long flags;
  891. s64 sdiff;
  892. raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
  893. offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
  894. ns = get_kernel_ns();
  895. elapsed = ns - kvm->arch.last_tsc_nsec;
  896. sdiff = data - kvm->arch.last_tsc_write;
  897. if (sdiff < 0)
  898. sdiff = -sdiff;
  899. /*
  900. * Special case: close write to TSC within 5 seconds of
  901. * another CPU is interpreted as an attempt to synchronize
  902. * The 5 seconds is to accommodate host load / swapping as
  903. * well as any reset of TSC during the boot process.
  904. *
  905. * In that case, for a reliable TSC, we can match TSC offsets,
  906. * or make a best guest using elapsed value.
  907. */
  908. if (sdiff < nsec_to_cycles(vcpu, 5ULL * NSEC_PER_SEC) &&
  909. elapsed < 5ULL * NSEC_PER_SEC) {
  910. if (!check_tsc_unstable()) {
  911. offset = kvm->arch.last_tsc_offset;
  912. pr_debug("kvm: matched tsc offset for %llu\n", data);
  913. } else {
  914. u64 delta = nsec_to_cycles(vcpu, elapsed);
  915. offset += delta;
  916. pr_debug("kvm: adjusted tsc offset by %llu\n", delta);
  917. }
  918. ns = kvm->arch.last_tsc_nsec;
  919. }
  920. kvm->arch.last_tsc_nsec = ns;
  921. kvm->arch.last_tsc_write = data;
  922. kvm->arch.last_tsc_offset = offset;
  923. kvm_x86_ops->write_tsc_offset(vcpu, offset);
  924. raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
  925. /* Reset of TSC must disable overshoot protection below */
  926. vcpu->arch.hv_clock.tsc_timestamp = 0;
  927. vcpu->arch.last_tsc_write = data;
  928. vcpu->arch.last_tsc_nsec = ns;
  929. }
  930. EXPORT_SYMBOL_GPL(kvm_write_tsc);
  931. static int kvm_guest_time_update(struct kvm_vcpu *v)
  932. {
  933. unsigned long flags;
  934. struct kvm_vcpu_arch *vcpu = &v->arch;
  935. void *shared_kaddr;
  936. unsigned long this_tsc_khz;
  937. s64 kernel_ns, max_kernel_ns;
  938. u64 tsc_timestamp;
  939. /* Keep irq disabled to prevent changes to the clock */
  940. local_irq_save(flags);
  941. kvm_get_msr(v, MSR_IA32_TSC, &tsc_timestamp);
  942. kernel_ns = get_kernel_ns();
  943. this_tsc_khz = vcpu_tsc_khz(v);
  944. if (unlikely(this_tsc_khz == 0)) {
  945. local_irq_restore(flags);
  946. kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
  947. return 1;
  948. }
  949. /*
  950. * We may have to catch up the TSC to match elapsed wall clock
  951. * time for two reasons, even if kvmclock is used.
  952. * 1) CPU could have been running below the maximum TSC rate
  953. * 2) Broken TSC compensation resets the base at each VCPU
  954. * entry to avoid unknown leaps of TSC even when running
  955. * again on the same CPU. This may cause apparent elapsed
  956. * time to disappear, and the guest to stand still or run
  957. * very slowly.
  958. */
  959. if (vcpu->tsc_catchup) {
  960. u64 tsc = compute_guest_tsc(v, kernel_ns);
  961. if (tsc > tsc_timestamp) {
  962. kvm_x86_ops->adjust_tsc_offset(v, tsc - tsc_timestamp);
  963. tsc_timestamp = tsc;
  964. }
  965. }
  966. local_irq_restore(flags);
  967. if (!vcpu->time_page)
  968. return 0;
  969. /*
  970. * Time as measured by the TSC may go backwards when resetting the base
  971. * tsc_timestamp. The reason for this is that the TSC resolution is
  972. * higher than the resolution of the other clock scales. Thus, many
  973. * possible measurments of the TSC correspond to one measurement of any
  974. * other clock, and so a spread of values is possible. This is not a
  975. * problem for the computation of the nanosecond clock; with TSC rates
  976. * around 1GHZ, there can only be a few cycles which correspond to one
  977. * nanosecond value, and any path through this code will inevitably
  978. * take longer than that. However, with the kernel_ns value itself,
  979. * the precision may be much lower, down to HZ granularity. If the
  980. * first sampling of TSC against kernel_ns ends in the low part of the
  981. * range, and the second in the high end of the range, we can get:
  982. *
  983. * (TSC - offset_low) * S + kns_old > (TSC - offset_high) * S + kns_new
  984. *
  985. * As the sampling errors potentially range in the thousands of cycles,
  986. * it is possible such a time value has already been observed by the
  987. * guest. To protect against this, we must compute the system time as
  988. * observed by the guest and ensure the new system time is greater.
  989. */
  990. max_kernel_ns = 0;
  991. if (vcpu->hv_clock.tsc_timestamp && vcpu->last_guest_tsc) {
  992. max_kernel_ns = vcpu->last_guest_tsc -
  993. vcpu->hv_clock.tsc_timestamp;
  994. max_kernel_ns = pvclock_scale_delta(max_kernel_ns,
  995. vcpu->hv_clock.tsc_to_system_mul,
  996. vcpu->hv_clock.tsc_shift);
  997. max_kernel_ns += vcpu->last_kernel_ns;
  998. }
  999. if (unlikely(vcpu->hw_tsc_khz != this_tsc_khz)) {
  1000. kvm_get_time_scale(NSEC_PER_SEC / 1000, this_tsc_khz,
  1001. &vcpu->hv_clock.tsc_shift,
  1002. &vcpu->hv_clock.tsc_to_system_mul);
  1003. vcpu->hw_tsc_khz = this_tsc_khz;
  1004. }
  1005. if (max_kernel_ns > kernel_ns)
  1006. kernel_ns = max_kernel_ns;
  1007. /* With all the info we got, fill in the values */
  1008. vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
  1009. vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
  1010. vcpu->last_kernel_ns = kernel_ns;
  1011. vcpu->last_guest_tsc = tsc_timestamp;
  1012. vcpu->hv_clock.flags = 0;
  1013. /*
  1014. * The interface expects us to write an even number signaling that the
  1015. * update is finished. Since the guest won't see the intermediate
  1016. * state, we just increase by 2 at the end.
  1017. */
  1018. vcpu->hv_clock.version += 2;
  1019. shared_kaddr = kmap_atomic(vcpu->time_page, KM_USER0);
  1020. memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock,
  1021. sizeof(vcpu->hv_clock));
  1022. kunmap_atomic(shared_kaddr, KM_USER0);
  1023. mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT);
  1024. return 0;
  1025. }
  1026. static bool msr_mtrr_valid(unsigned msr)
  1027. {
  1028. switch (msr) {
  1029. case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
  1030. case MSR_MTRRfix64K_00000:
  1031. case MSR_MTRRfix16K_80000:
  1032. case MSR_MTRRfix16K_A0000:
  1033. case MSR_MTRRfix4K_C0000:
  1034. case MSR_MTRRfix4K_C8000:
  1035. case MSR_MTRRfix4K_D0000:
  1036. case MSR_MTRRfix4K_D8000:
  1037. case MSR_MTRRfix4K_E0000:
  1038. case MSR_MTRRfix4K_E8000:
  1039. case MSR_MTRRfix4K_F0000:
  1040. case MSR_MTRRfix4K_F8000:
  1041. case MSR_MTRRdefType:
  1042. case MSR_IA32_CR_PAT:
  1043. return true;
  1044. case 0x2f8:
  1045. return true;
  1046. }
  1047. return false;
  1048. }
  1049. static bool valid_pat_type(unsigned t)
  1050. {
  1051. return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
  1052. }
  1053. static bool valid_mtrr_type(unsigned t)
  1054. {
  1055. return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
  1056. }
  1057. static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1058. {
  1059. int i;
  1060. if (!msr_mtrr_valid(msr))
  1061. return false;
  1062. if (msr == MSR_IA32_CR_PAT) {
  1063. for (i = 0; i < 8; i++)
  1064. if (!valid_pat_type((data >> (i * 8)) & 0xff))
  1065. return false;
  1066. return true;
  1067. } else if (msr == MSR_MTRRdefType) {
  1068. if (data & ~0xcff)
  1069. return false;
  1070. return valid_mtrr_type(data & 0xff);
  1071. } else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
  1072. for (i = 0; i < 8 ; i++)
  1073. if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
  1074. return false;
  1075. return true;
  1076. }
  1077. /* variable MTRRs */
  1078. return valid_mtrr_type(data & 0xff);
  1079. }
  1080. static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1081. {
  1082. u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
  1083. if (!mtrr_valid(vcpu, msr, data))
  1084. return 1;
  1085. if (msr == MSR_MTRRdefType) {
  1086. vcpu->arch.mtrr_state.def_type = data;
  1087. vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
  1088. } else if (msr == MSR_MTRRfix64K_00000)
  1089. p[0] = data;
  1090. else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
  1091. p[1 + msr - MSR_MTRRfix16K_80000] = data;
  1092. else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
  1093. p[3 + msr - MSR_MTRRfix4K_C0000] = data;
  1094. else if (msr == MSR_IA32_CR_PAT)
  1095. vcpu->arch.pat = data;
  1096. else { /* Variable MTRRs */
  1097. int idx, is_mtrr_mask;
  1098. u64 *pt;
  1099. idx = (msr - 0x200) / 2;
  1100. is_mtrr_mask = msr - 0x200 - 2 * idx;
  1101. if (!is_mtrr_mask)
  1102. pt =
  1103. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
  1104. else
  1105. pt =
  1106. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
  1107. *pt = data;
  1108. }
  1109. kvm_mmu_reset_context(vcpu);
  1110. return 0;
  1111. }
  1112. static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1113. {
  1114. u64 mcg_cap = vcpu->arch.mcg_cap;
  1115. unsigned bank_num = mcg_cap & 0xff;
  1116. switch (msr) {
  1117. case MSR_IA32_MCG_STATUS:
  1118. vcpu->arch.mcg_status = data;
  1119. break;
  1120. case MSR_IA32_MCG_CTL:
  1121. if (!(mcg_cap & MCG_CTL_P))
  1122. return 1;
  1123. if (data != 0 && data != ~(u64)0)
  1124. return -1;
  1125. vcpu->arch.mcg_ctl = data;
  1126. break;
  1127. default:
  1128. if (msr >= MSR_IA32_MC0_CTL &&
  1129. msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
  1130. u32 offset = msr - MSR_IA32_MC0_CTL;
  1131. /* only 0 or all 1s can be written to IA32_MCi_CTL
  1132. * some Linux kernels though clear bit 10 in bank 4 to
  1133. * workaround a BIOS/GART TBL issue on AMD K8s, ignore
  1134. * this to avoid an uncatched #GP in the guest
  1135. */
  1136. if ((offset & 0x3) == 0 &&
  1137. data != 0 && (data | (1 << 10)) != ~(u64)0)
  1138. return -1;
  1139. vcpu->arch.mce_banks[offset] = data;
  1140. break;
  1141. }
  1142. return 1;
  1143. }
  1144. return 0;
  1145. }
  1146. static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
  1147. {
  1148. struct kvm *kvm = vcpu->kvm;
  1149. int lm = is_long_mode(vcpu);
  1150. u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
  1151. : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
  1152. u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
  1153. : kvm->arch.xen_hvm_config.blob_size_32;
  1154. u32 page_num = data & ~PAGE_MASK;
  1155. u64 page_addr = data & PAGE_MASK;
  1156. u8 *page;
  1157. int r;
  1158. r = -E2BIG;
  1159. if (page_num >= blob_size)
  1160. goto out;
  1161. r = -ENOMEM;
  1162. page = kzalloc(PAGE_SIZE, GFP_KERNEL);
  1163. if (!page)
  1164. goto out;
  1165. r = -EFAULT;
  1166. if (copy_from_user(page, blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE))
  1167. goto out_free;
  1168. if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
  1169. goto out_free;
  1170. r = 0;
  1171. out_free:
  1172. kfree(page);
  1173. out:
  1174. return r;
  1175. }
  1176. static bool kvm_hv_hypercall_enabled(struct kvm *kvm)
  1177. {
  1178. return kvm->arch.hv_hypercall & HV_X64_MSR_HYPERCALL_ENABLE;
  1179. }
  1180. static bool kvm_hv_msr_partition_wide(u32 msr)
  1181. {
  1182. bool r = false;
  1183. switch (msr) {
  1184. case HV_X64_MSR_GUEST_OS_ID:
  1185. case HV_X64_MSR_HYPERCALL:
  1186. r = true;
  1187. break;
  1188. }
  1189. return r;
  1190. }
  1191. static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1192. {
  1193. struct kvm *kvm = vcpu->kvm;
  1194. switch (msr) {
  1195. case HV_X64_MSR_GUEST_OS_ID:
  1196. kvm->arch.hv_guest_os_id = data;
  1197. /* setting guest os id to zero disables hypercall page */
  1198. if (!kvm->arch.hv_guest_os_id)
  1199. kvm->arch.hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
  1200. break;
  1201. case HV_X64_MSR_HYPERCALL: {
  1202. u64 gfn;
  1203. unsigned long addr;
  1204. u8 instructions[4];
  1205. /* if guest os id is not set hypercall should remain disabled */
  1206. if (!kvm->arch.hv_guest_os_id)
  1207. break;
  1208. if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) {
  1209. kvm->arch.hv_hypercall = data;
  1210. break;
  1211. }
  1212. gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;
  1213. addr = gfn_to_hva(kvm, gfn);
  1214. if (kvm_is_error_hva(addr))
  1215. return 1;
  1216. kvm_x86_ops->patch_hypercall(vcpu, instructions);
  1217. ((unsigned char *)instructions)[3] = 0xc3; /* ret */
  1218. if (__copy_to_user((void __user *)addr, instructions, 4))
  1219. return 1;
  1220. kvm->arch.hv_hypercall = data;
  1221. break;
  1222. }
  1223. default:
  1224. pr_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
  1225. "data 0x%llx\n", msr, data);
  1226. return 1;
  1227. }
  1228. return 0;
  1229. }
  1230. static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1231. {
  1232. switch (msr) {
  1233. case HV_X64_MSR_APIC_ASSIST_PAGE: {
  1234. unsigned long addr;
  1235. if (!(data & HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE)) {
  1236. vcpu->arch.hv_vapic = data;
  1237. break;
  1238. }
  1239. addr = gfn_to_hva(vcpu->kvm, data >>
  1240. HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT);
  1241. if (kvm_is_error_hva(addr))
  1242. return 1;
  1243. if (__clear_user((void __user *)addr, PAGE_SIZE))
  1244. return 1;
  1245. vcpu->arch.hv_vapic = data;
  1246. break;
  1247. }
  1248. case HV_X64_MSR_EOI:
  1249. return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data);
  1250. case HV_X64_MSR_ICR:
  1251. return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data);
  1252. case HV_X64_MSR_TPR:
  1253. return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data);
  1254. default:
  1255. pr_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
  1256. "data 0x%llx\n", msr, data);
  1257. return 1;
  1258. }
  1259. return 0;
  1260. }
  1261. static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
  1262. {
  1263. gpa_t gpa = data & ~0x3f;
  1264. /* Bits 2:5 are resrved, Should be zero */
  1265. if (data & 0x3c)
  1266. return 1;
  1267. vcpu->arch.apf.msr_val = data;
  1268. if (!(data & KVM_ASYNC_PF_ENABLED)) {
  1269. kvm_clear_async_pf_completion_queue(vcpu);
  1270. kvm_async_pf_hash_reset(vcpu);
  1271. return 0;
  1272. }
  1273. if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa))
  1274. return 1;
  1275. vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
  1276. kvm_async_pf_wakeup_all(vcpu);
  1277. return 0;
  1278. }
  1279. static void kvmclock_reset(struct kvm_vcpu *vcpu)
  1280. {
  1281. if (vcpu->arch.time_page) {
  1282. kvm_release_page_dirty(vcpu->arch.time_page);
  1283. vcpu->arch.time_page = NULL;
  1284. }
  1285. }
  1286. static void accumulate_steal_time(struct kvm_vcpu *vcpu)
  1287. {
  1288. u64 delta;
  1289. if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
  1290. return;
  1291. delta = current->sched_info.run_delay - vcpu->arch.st.last_steal;
  1292. vcpu->arch.st.last_steal = current->sched_info.run_delay;
  1293. vcpu->arch.st.accum_steal = delta;
  1294. }
  1295. static void record_steal_time(struct kvm_vcpu *vcpu)
  1296. {
  1297. if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
  1298. return;
  1299. if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
  1300. &vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
  1301. return;
  1302. vcpu->arch.st.steal.steal += vcpu->arch.st.accum_steal;
  1303. vcpu->arch.st.steal.version += 2;
  1304. vcpu->arch.st.accum_steal = 0;
  1305. kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
  1306. &vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
  1307. }
  1308. int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1309. {
  1310. switch (msr) {
  1311. case MSR_EFER:
  1312. return set_efer(vcpu, data);
  1313. case MSR_K7_HWCR:
  1314. data &= ~(u64)0x40; /* ignore flush filter disable */
  1315. data &= ~(u64)0x100; /* ignore ignne emulation enable */
  1316. if (data != 0) {
  1317. pr_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
  1318. data);
  1319. return 1;
  1320. }
  1321. break;
  1322. case MSR_FAM10H_MMIO_CONF_BASE:
  1323. if (data != 0) {
  1324. pr_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
  1325. "0x%llx\n", data);
  1326. return 1;
  1327. }
  1328. break;
  1329. case MSR_AMD64_NB_CFG:
  1330. break;
  1331. case MSR_IA32_DEBUGCTLMSR:
  1332. if (!data) {
  1333. /* We support the non-activated case already */
  1334. break;
  1335. } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
  1336. /* Values other than LBR and BTF are vendor-specific,
  1337. thus reserved and should throw a #GP */
  1338. return 1;
  1339. }
  1340. pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
  1341. __func__, data);
  1342. break;
  1343. case MSR_IA32_UCODE_REV:
  1344. case MSR_IA32_UCODE_WRITE:
  1345. case MSR_VM_HSAVE_PA:
  1346. case MSR_AMD64_PATCH_LOADER:
  1347. break;
  1348. case 0x200 ... 0x2ff:
  1349. return set_msr_mtrr(vcpu, msr, data);
  1350. case MSR_IA32_APICBASE:
  1351. kvm_set_apic_base(vcpu, data);
  1352. break;
  1353. case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
  1354. return kvm_x2apic_msr_write(vcpu, msr, data);
  1355. case MSR_IA32_MISC_ENABLE:
  1356. vcpu->arch.ia32_misc_enable_msr = data;
  1357. break;
  1358. case MSR_KVM_WALL_CLOCK_NEW:
  1359. case MSR_KVM_WALL_CLOCK:
  1360. vcpu->kvm->arch.wall_clock = data;
  1361. kvm_write_wall_clock(vcpu->kvm, data);
  1362. break;
  1363. case MSR_KVM_SYSTEM_TIME_NEW:
  1364. case MSR_KVM_SYSTEM_TIME: {
  1365. kvmclock_reset(vcpu);
  1366. vcpu->arch.time = data;
  1367. kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
  1368. /* we verify if the enable bit is set... */
  1369. if (!(data & 1))
  1370. break;
  1371. /* ...but clean it before doing the actual write */
  1372. vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);
  1373. vcpu->arch.time_page =
  1374. gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);
  1375. if (is_error_page(vcpu->arch.time_page)) {
  1376. kvm_release_page_clean(vcpu->arch.time_page);
  1377. vcpu->arch.time_page = NULL;
  1378. }
  1379. break;
  1380. }
  1381. case MSR_KVM_ASYNC_PF_EN:
  1382. if (kvm_pv_enable_async_pf(vcpu, data))
  1383. return 1;
  1384. break;
  1385. case MSR_KVM_STEAL_TIME:
  1386. if (unlikely(!sched_info_on()))
  1387. return 1;
  1388. if (data & KVM_STEAL_RESERVED_MASK)
  1389. return 1;
  1390. if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
  1391. data & KVM_STEAL_VALID_BITS))
  1392. return 1;
  1393. vcpu->arch.st.msr_val = data;
  1394. if (!(data & KVM_MSR_ENABLED))
  1395. break;
  1396. vcpu->arch.st.last_steal = current->sched_info.run_delay;
  1397. preempt_disable();
  1398. accumulate_steal_time(vcpu);
  1399. preempt_enable();
  1400. kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
  1401. break;
  1402. case MSR_IA32_MCG_CTL:
  1403. case MSR_IA32_MCG_STATUS:
  1404. case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
  1405. return set_msr_mce(vcpu, msr, data);
  1406. /* Performance counters are not protected by a CPUID bit,
  1407. * so we should check all of them in the generic path for the sake of
  1408. * cross vendor migration.
  1409. * Writing a zero into the event select MSRs disables them,
  1410. * which we perfectly emulate ;-). Any other value should be at least
  1411. * reported, some guests depend on them.
  1412. */
  1413. case MSR_P6_EVNTSEL0:
  1414. case MSR_P6_EVNTSEL1:
  1415. case MSR_K7_EVNTSEL0:
  1416. case MSR_K7_EVNTSEL1:
  1417. case MSR_K7_EVNTSEL2:
  1418. case MSR_K7_EVNTSEL3:
  1419. if (data != 0)
  1420. pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
  1421. "0x%x data 0x%llx\n", msr, data);
  1422. break;
  1423. /* at least RHEL 4 unconditionally writes to the perfctr registers,
  1424. * so we ignore writes to make it happy.
  1425. */
  1426. case MSR_P6_PERFCTR0:
  1427. case MSR_P6_PERFCTR1:
  1428. case MSR_K7_PERFCTR0:
  1429. case MSR_K7_PERFCTR1:
  1430. case MSR_K7_PERFCTR2:
  1431. case MSR_K7_PERFCTR3:
  1432. pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
  1433. "0x%x data 0x%llx\n", msr, data);
  1434. break;
  1435. case MSR_K7_CLK_CTL:
  1436. /*
  1437. * Ignore all writes to this no longer documented MSR.
  1438. * Writes are only relevant for old K7 processors,
  1439. * all pre-dating SVM, but a recommended workaround from
  1440. * AMD for these chips. It is possible to speicify the
  1441. * affected processor models on the command line, hence
  1442. * the need to ignore the workaround.
  1443. */
  1444. break;
  1445. case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
  1446. if (kvm_hv_msr_partition_wide(msr)) {
  1447. int r;
  1448. mutex_lock(&vcpu->kvm->lock);
  1449. r = set_msr_hyperv_pw(vcpu, msr, data);
  1450. mutex_unlock(&vcpu->kvm->lock);
  1451. return r;
  1452. } else
  1453. return set_msr_hyperv(vcpu, msr, data);
  1454. break;
  1455. case MSR_IA32_BBL_CR_CTL3:
  1456. /* Drop writes to this legacy MSR -- see rdmsr
  1457. * counterpart for further detail.
  1458. */
  1459. pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", msr, data);
  1460. break;
  1461. default:
  1462. if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
  1463. return xen_hvm_config(vcpu, data);
  1464. if (!ignore_msrs) {
  1465. pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
  1466. msr, data);
  1467. return 1;
  1468. } else {
  1469. pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
  1470. msr, data);
  1471. break;
  1472. }
  1473. }
  1474. return 0;
  1475. }
  1476. EXPORT_SYMBOL_GPL(kvm_set_msr_common);
  1477. /*
  1478. * Reads an msr value (of 'msr_index') into 'pdata'.
  1479. * Returns 0 on success, non-0 otherwise.
  1480. * Assumes vcpu_load() was already called.
  1481. */
  1482. int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
  1483. {
  1484. return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
  1485. }
  1486. static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1487. {
  1488. u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
  1489. if (!msr_mtrr_valid(msr))
  1490. return 1;
  1491. if (msr == MSR_MTRRdefType)
  1492. *pdata = vcpu->arch.mtrr_state.def_type +
  1493. (vcpu->arch.mtrr_state.enabled << 10);
  1494. else if (msr == MSR_MTRRfix64K_00000)
  1495. *pdata = p[0];
  1496. else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
  1497. *pdata = p[1 + msr - MSR_MTRRfix16K_80000];
  1498. else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
  1499. *pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
  1500. else if (msr == MSR_IA32_CR_PAT)
  1501. *pdata = vcpu->arch.pat;
  1502. else { /* Variable MTRRs */
  1503. int idx, is_mtrr_mask;
  1504. u64 *pt;
  1505. idx = (msr - 0x200) / 2;
  1506. is_mtrr_mask = msr - 0x200 - 2 * idx;
  1507. if (!is_mtrr_mask)
  1508. pt =
  1509. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
  1510. else
  1511. pt =
  1512. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
  1513. *pdata = *pt;
  1514. }
  1515. return 0;
  1516. }
  1517. static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1518. {
  1519. u64 data;
  1520. u64 mcg_cap = vcpu->arch.mcg_cap;
  1521. unsigned bank_num = mcg_cap & 0xff;
  1522. switch (msr) {
  1523. case MSR_IA32_P5_MC_ADDR:
  1524. case MSR_IA32_P5_MC_TYPE:
  1525. data = 0;
  1526. break;
  1527. case MSR_IA32_MCG_CAP:
  1528. data = vcpu->arch.mcg_cap;
  1529. break;
  1530. case MSR_IA32_MCG_CTL:
  1531. if (!(mcg_cap & MCG_CTL_P))
  1532. return 1;
  1533. data = vcpu->arch.mcg_ctl;
  1534. break;
  1535. case MSR_IA32_MCG_STATUS:
  1536. data = vcpu->arch.mcg_status;
  1537. break;
  1538. default:
  1539. if (msr >= MSR_IA32_MC0_CTL &&
  1540. msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
  1541. u32 offset = msr - MSR_IA32_MC0_CTL;
  1542. data = vcpu->arch.mce_banks[offset];
  1543. break;
  1544. }
  1545. return 1;
  1546. }
  1547. *pdata = data;
  1548. return 0;
  1549. }
  1550. static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1551. {
  1552. u64 data = 0;
  1553. struct kvm *kvm = vcpu->kvm;
  1554. switch (msr) {
  1555. case HV_X64_MSR_GUEST_OS_ID:
  1556. data = kvm->arch.hv_guest_os_id;
  1557. break;
  1558. case HV_X64_MSR_HYPERCALL:
  1559. data = kvm->arch.hv_hypercall;
  1560. break;
  1561. default:
  1562. pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
  1563. return 1;
  1564. }
  1565. *pdata = data;
  1566. return 0;
  1567. }
  1568. static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1569. {
  1570. u64 data = 0;
  1571. switch (msr) {
  1572. case HV_X64_MSR_VP_INDEX: {
  1573. int r;
  1574. struct kvm_vcpu *v;
  1575. kvm_for_each_vcpu(r, v, vcpu->kvm)
  1576. if (v == vcpu)
  1577. data = r;
  1578. break;
  1579. }
  1580. case HV_X64_MSR_EOI:
  1581. return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata);
  1582. case HV_X64_MSR_ICR:
  1583. return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
  1584. case HV_X64_MSR_TPR:
  1585. return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
  1586. default:
  1587. pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
  1588. return 1;
  1589. }
  1590. *pdata = data;
  1591. return 0;
  1592. }
  1593. int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1594. {
  1595. u64 data;
  1596. switch (msr) {
  1597. case MSR_IA32_PLATFORM_ID:
  1598. case MSR_IA32_UCODE_REV:
  1599. case MSR_IA32_EBL_CR_POWERON:
  1600. case MSR_IA32_DEBUGCTLMSR:
  1601. case MSR_IA32_LASTBRANCHFROMIP:
  1602. case MSR_IA32_LASTBRANCHTOIP:
  1603. case MSR_IA32_LASTINTFROMIP:
  1604. case MSR_IA32_LASTINTTOIP:
  1605. case MSR_K8_SYSCFG:
  1606. case MSR_K7_HWCR:
  1607. case MSR_VM_HSAVE_PA:
  1608. case MSR_P6_PERFCTR0:
  1609. case MSR_P6_PERFCTR1:
  1610. case MSR_P6_EVNTSEL0:
  1611. case MSR_P6_EVNTSEL1:
  1612. case MSR_K7_EVNTSEL0:
  1613. case MSR_K7_PERFCTR0:
  1614. case MSR_K8_INT_PENDING_MSG:
  1615. case MSR_AMD64_NB_CFG:
  1616. case MSR_FAM10H_MMIO_CONF_BASE:
  1617. data = 0;
  1618. break;
  1619. case MSR_MTRRcap:
  1620. data = 0x500 | KVM_NR_VAR_MTRR;
  1621. break;
  1622. case 0x200 ... 0x2ff:
  1623. return get_msr_mtrr(vcpu, msr, pdata);
  1624. case 0xcd: /* fsb frequency */
  1625. data = 3;
  1626. break;
  1627. /*
  1628. * MSR_EBC_FREQUENCY_ID
  1629. * Conservative value valid for even the basic CPU models.
  1630. * Models 0,1: 000 in bits 23:21 indicating a bus speed of
  1631. * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
  1632. * and 266MHz for model 3, or 4. Set Core Clock
  1633. * Frequency to System Bus Frequency Ratio to 1 (bits
  1634. * 31:24) even though these are only valid for CPU
  1635. * models > 2, however guests may end up dividing or
  1636. * multiplying by zero otherwise.
  1637. */
  1638. case MSR_EBC_FREQUENCY_ID:
  1639. data = 1 << 24;
  1640. break;
  1641. case MSR_IA32_APICBASE:
  1642. data = kvm_get_apic_base(vcpu);
  1643. break;
  1644. case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
  1645. return kvm_x2apic_msr_read(vcpu, msr, pdata);
  1646. break;
  1647. case MSR_IA32_MISC_ENABLE:
  1648. data = vcpu->arch.ia32_misc_enable_msr;
  1649. break;
  1650. case MSR_IA32_PERF_STATUS:
  1651. /* TSC increment by tick */
  1652. data = 1000ULL;
  1653. /* CPU multiplier */
  1654. data |= (((uint64_t)4ULL) << 40);
  1655. break;
  1656. case MSR_EFER:
  1657. data = vcpu->arch.efer;
  1658. break;
  1659. case MSR_KVM_WALL_CLOCK:
  1660. case MSR_KVM_WALL_CLOCK_NEW:
  1661. data = vcpu->kvm->arch.wall_clock;
  1662. break;
  1663. case MSR_KVM_SYSTEM_TIME:
  1664. case MSR_KVM_SYSTEM_TIME_NEW:
  1665. data = vcpu->arch.time;
  1666. break;
  1667. case MSR_KVM_ASYNC_PF_EN:
  1668. data = vcpu->arch.apf.msr_val;
  1669. break;
  1670. case MSR_KVM_STEAL_TIME:
  1671. data = vcpu->arch.st.msr_val;
  1672. break;
  1673. case MSR_IA32_P5_MC_ADDR:
  1674. case MSR_IA32_P5_MC_TYPE:
  1675. case MSR_IA32_MCG_CAP:
  1676. case MSR_IA32_MCG_CTL:
  1677. case MSR_IA32_MCG_STATUS:
  1678. case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
  1679. return get_msr_mce(vcpu, msr, pdata);
  1680. case MSR_K7_CLK_CTL:
  1681. /*
  1682. * Provide expected ramp-up count for K7. All other
  1683. * are set to zero, indicating minimum divisors for
  1684. * every field.
  1685. *
  1686. * This prevents guest kernels on AMD host with CPU
  1687. * type 6, model 8 and higher from exploding due to
  1688. * the rdmsr failing.
  1689. */
  1690. data = 0x20000000;
  1691. break;
  1692. case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
  1693. if (kvm_hv_msr_partition_wide(msr)) {
  1694. int r;
  1695. mutex_lock(&vcpu->kvm->lock);
  1696. r = get_msr_hyperv_pw(vcpu, msr, pdata);
  1697. mutex_unlock(&vcpu->kvm->lock);
  1698. return r;
  1699. } else
  1700. return get_msr_hyperv(vcpu, msr, pdata);
  1701. break;
  1702. case MSR_IA32_BBL_CR_CTL3:
  1703. /* This legacy MSR exists but isn't fully documented in current
  1704. * silicon. It is however accessed by winxp in very narrow
  1705. * scenarios where it sets bit #19, itself documented as
  1706. * a "reserved" bit. Best effort attempt to source coherent
  1707. * read data here should the balance of the register be
  1708. * interpreted by the guest:
  1709. *
  1710. * L2 cache control register 3: 64GB range, 256KB size,
  1711. * enabled, latency 0x1, configured
  1712. */
  1713. data = 0xbe702111;
  1714. break;
  1715. default:
  1716. if (!ignore_msrs) {
  1717. pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
  1718. return 1;
  1719. } else {
  1720. pr_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
  1721. data = 0;
  1722. }
  1723. break;
  1724. }
  1725. *pdata = data;
  1726. return 0;
  1727. }
  1728. EXPORT_SYMBOL_GPL(kvm_get_msr_common);
  1729. /*
  1730. * Read or write a bunch of msrs. All parameters are kernel addresses.
  1731. *
  1732. * @return number of msrs set successfully.
  1733. */
  1734. static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
  1735. struct kvm_msr_entry *entries,
  1736. int (*do_msr)(struct kvm_vcpu *vcpu,
  1737. unsigned index, u64 *data))
  1738. {
  1739. int i, idx;
  1740. idx = srcu_read_lock(&vcpu->kvm->srcu);
  1741. for (i = 0; i < msrs->nmsrs; ++i)
  1742. if (do_msr(vcpu, entries[i].index, &entries[i].data))
  1743. break;
  1744. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  1745. return i;
  1746. }
  1747. /*
  1748. * Read or write a bunch of msrs. Parameters are user addresses.
  1749. *
  1750. * @return number of msrs set successfully.
  1751. */
  1752. static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
  1753. int (*do_msr)(struct kvm_vcpu *vcpu,
  1754. unsigned index, u64 *data),
  1755. int writeback)
  1756. {
  1757. struct kvm_msrs msrs;
  1758. struct kvm_msr_entry *entries;
  1759. int r, n;
  1760. unsigned size;
  1761. r = -EFAULT;
  1762. if (copy_from_user(&msrs, user_msrs, sizeof msrs))
  1763. goto out;
  1764. r = -E2BIG;
  1765. if (msrs.nmsrs >= MAX_IO_MSRS)
  1766. goto out;
  1767. r = -ENOMEM;
  1768. size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
  1769. entries = kmalloc(size, GFP_KERNEL);
  1770. if (!entries)
  1771. goto out;
  1772. r = -EFAULT;
  1773. if (copy_from_user(entries, user_msrs->entries, size))
  1774. goto out_free;
  1775. r = n = __msr_io(vcpu, &msrs, entries, do_msr);
  1776. if (r < 0)
  1777. goto out_free;
  1778. r = -EFAULT;
  1779. if (writeback && copy_to_user(user_msrs->entries, entries, size))
  1780. goto out_free;
  1781. r = n;
  1782. out_free:
  1783. kfree(entries);
  1784. out:
  1785. return r;
  1786. }
  1787. int kvm_dev_ioctl_check_extension(long ext)
  1788. {
  1789. int r;
  1790. switch (ext) {
  1791. case KVM_CAP_IRQCHIP:
  1792. case KVM_CAP_HLT:
  1793. case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
  1794. case KVM_CAP_SET_TSS_ADDR:
  1795. case KVM_CAP_EXT_CPUID:
  1796. case KVM_CAP_CLOCKSOURCE:
  1797. case KVM_CAP_PIT:
  1798. case KVM_CAP_NOP_IO_DELAY:
  1799. case KVM_CAP_MP_STATE:
  1800. case KVM_CAP_SYNC_MMU:
  1801. case KVM_CAP_USER_NMI:
  1802. case KVM_CAP_REINJECT_CONTROL:
  1803. case KVM_CAP_IRQ_INJECT_STATUS:
  1804. case KVM_CAP_ASSIGN_DEV_IRQ:
  1805. case KVM_CAP_IRQFD:
  1806. case KVM_CAP_IOEVENTFD:
  1807. case KVM_CAP_PIT2:
  1808. case KVM_CAP_PIT_STATE2:
  1809. case KVM_CAP_SET_IDENTITY_MAP_ADDR:
  1810. case KVM_CAP_XEN_HVM:
  1811. case KVM_CAP_ADJUST_CLOCK:
  1812. case KVM_CAP_VCPU_EVENTS:
  1813. case KVM_CAP_HYPERV:
  1814. case KVM_CAP_HYPERV_VAPIC:
  1815. case KVM_CAP_HYPERV_SPIN:
  1816. case KVM_CAP_PCI_SEGMENT:
  1817. case KVM_CAP_DEBUGREGS:
  1818. case KVM_CAP_X86_ROBUST_SINGLESTEP:
  1819. case KVM_CAP_XSAVE:
  1820. case KVM_CAP_ASYNC_PF:
  1821. case KVM_CAP_GET_TSC_KHZ:
  1822. r = 1;
  1823. break;
  1824. case KVM_CAP_COALESCED_MMIO:
  1825. r = KVM_COALESCED_MMIO_PAGE_OFFSET;
  1826. break;
  1827. case KVM_CAP_VAPIC:
  1828. r = !kvm_x86_ops->cpu_has_accelerated_tpr();
  1829. break;
  1830. case KVM_CAP_NR_VCPUS:
  1831. r = KVM_SOFT_MAX_VCPUS;
  1832. break;
  1833. case KVM_CAP_MAX_VCPUS:
  1834. r = KVM_MAX_VCPUS;
  1835. break;
  1836. case KVM_CAP_NR_MEMSLOTS:
  1837. r = KVM_MEMORY_SLOTS;
  1838. break;
  1839. case KVM_CAP_PV_MMU: /* obsolete */
  1840. r = 0;
  1841. break;
  1842. case KVM_CAP_IOMMU:
  1843. r = iommu_found();
  1844. break;
  1845. case KVM_CAP_MCE:
  1846. r = KVM_MAX_MCE_BANKS;
  1847. break;
  1848. case KVM_CAP_XCRS:
  1849. r = cpu_has_xsave;
  1850. break;
  1851. case KVM_CAP_TSC_CONTROL:
  1852. r = kvm_has_tsc_control;
  1853. break;
  1854. default:
  1855. r = 0;
  1856. break;
  1857. }
  1858. return r;
  1859. }
  1860. long kvm_arch_dev_ioctl(struct file *filp,
  1861. unsigned int ioctl, unsigned long arg)
  1862. {
  1863. void __user *argp = (void __user *)arg;
  1864. long r;
  1865. switch (ioctl) {
  1866. case KVM_GET_MSR_INDEX_LIST: {
  1867. struct kvm_msr_list __user *user_msr_list = argp;
  1868. struct kvm_msr_list msr_list;
  1869. unsigned n;
  1870. r = -EFAULT;
  1871. if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
  1872. goto out;
  1873. n = msr_list.nmsrs;
  1874. msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
  1875. if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
  1876. goto out;
  1877. r = -E2BIG;
  1878. if (n < msr_list.nmsrs)
  1879. goto out;
  1880. r = -EFAULT;
  1881. if (copy_to_user(user_msr_list->indices, &msrs_to_save,
  1882. num_msrs_to_save * sizeof(u32)))
  1883. goto out;
  1884. if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
  1885. &emulated_msrs,
  1886. ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
  1887. goto out;
  1888. r = 0;
  1889. break;
  1890. }
  1891. case KVM_GET_SUPPORTED_CPUID: {
  1892. struct kvm_cpuid2 __user *cpuid_arg = argp;
  1893. struct kvm_cpuid2 cpuid;
  1894. r = -EFAULT;
  1895. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1896. goto out;
  1897. r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
  1898. cpuid_arg->entries);
  1899. if (r)
  1900. goto out;
  1901. r = -EFAULT;
  1902. if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
  1903. goto out;
  1904. r = 0;
  1905. break;
  1906. }
  1907. case KVM_X86_GET_MCE_CAP_SUPPORTED: {
  1908. u64 mce_cap;
  1909. mce_cap = KVM_MCE_CAP_SUPPORTED;
  1910. r = -EFAULT;
  1911. if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
  1912. goto out;
  1913. r = 0;
  1914. break;
  1915. }
  1916. default:
  1917. r = -EINVAL;
  1918. }
  1919. out:
  1920. return r;
  1921. }
  1922. static void wbinvd_ipi(void *garbage)
  1923. {
  1924. wbinvd();
  1925. }
  1926. static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
  1927. {
  1928. return vcpu->kvm->arch.iommu_domain &&
  1929. !(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY);
  1930. }
  1931. void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  1932. {
  1933. /* Address WBINVD may be executed by guest */
  1934. if (need_emulate_wbinvd(vcpu)) {
  1935. if (kvm_x86_ops->has_wbinvd_exit())
  1936. cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
  1937. else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
  1938. smp_call_function_single(vcpu->cpu,
  1939. wbinvd_ipi, NULL, 1);
  1940. }
  1941. kvm_x86_ops->vcpu_load(vcpu, cpu);
  1942. if (unlikely(vcpu->cpu != cpu) || check_tsc_unstable()) {
  1943. /* Make sure TSC doesn't go backwards */
  1944. s64 tsc_delta;
  1945. u64 tsc;
  1946. kvm_get_msr(vcpu, MSR_IA32_TSC, &tsc);
  1947. tsc_delta = !vcpu->arch.last_guest_tsc ? 0 :
  1948. tsc - vcpu->arch.last_guest_tsc;
  1949. if (tsc_delta < 0)
  1950. mark_tsc_unstable("KVM discovered backwards TSC");
  1951. if (check_tsc_unstable()) {
  1952. kvm_x86_ops->adjust_tsc_offset(vcpu, -tsc_delta);
  1953. vcpu->arch.tsc_catchup = 1;
  1954. }
  1955. kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
  1956. if (vcpu->cpu != cpu)
  1957. kvm_migrate_timers(vcpu);
  1958. vcpu->cpu = cpu;
  1959. }
  1960. accumulate_steal_time(vcpu);
  1961. kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
  1962. }
  1963. void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
  1964. {
  1965. kvm_x86_ops->vcpu_put(vcpu);
  1966. kvm_put_guest_fpu(vcpu);
  1967. kvm_get_msr(vcpu, MSR_IA32_TSC, &vcpu->arch.last_guest_tsc);
  1968. }
  1969. static int is_efer_nx(void)
  1970. {
  1971. unsigned long long efer = 0;
  1972. rdmsrl_safe(MSR_EFER, &efer);
  1973. return efer & EFER_NX;
  1974. }
  1975. static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
  1976. {
  1977. int i;
  1978. struct kvm_cpuid_entry2 *e, *entry;
  1979. entry = NULL;
  1980. for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
  1981. e = &vcpu->arch.cpuid_entries[i];
  1982. if (e->function == 0x80000001) {
  1983. entry = e;
  1984. break;
  1985. }
  1986. }
  1987. if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) {
  1988. entry->edx &= ~(1 << 20);
  1989. printk(KERN_INFO "kvm: guest NX capability removed\n");
  1990. }
  1991. }
  1992. /* when an old userspace process fills a new kernel module */
  1993. static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
  1994. struct kvm_cpuid *cpuid,
  1995. struct kvm_cpuid_entry __user *entries)
  1996. {
  1997. int r, i;
  1998. struct kvm_cpuid_entry *cpuid_entries;
  1999. r = -E2BIG;
  2000. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  2001. goto out;
  2002. r = -ENOMEM;
  2003. cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent);
  2004. if (!cpuid_entries)
  2005. goto out;
  2006. r = -EFAULT;
  2007. if (copy_from_user(cpuid_entries, entries,
  2008. cpuid->nent * sizeof(struct kvm_cpuid_entry)))
  2009. goto out_free;
  2010. for (i = 0; i < cpuid->nent; i++) {
  2011. vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
  2012. vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
  2013. vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
  2014. vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
  2015. vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
  2016. vcpu->arch.cpuid_entries[i].index = 0;
  2017. vcpu->arch.cpuid_entries[i].flags = 0;
  2018. vcpu->arch.cpuid_entries[i].padding[0] = 0;
  2019. vcpu->arch.cpuid_entries[i].padding[1] = 0;
  2020. vcpu->arch.cpuid_entries[i].padding[2] = 0;
  2021. }
  2022. vcpu->arch.cpuid_nent = cpuid->nent;
  2023. cpuid_fix_nx_cap(vcpu);
  2024. r = 0;
  2025. kvm_apic_set_version(vcpu);
  2026. kvm_x86_ops->cpuid_update(vcpu);
  2027. update_cpuid(vcpu);
  2028. out_free:
  2029. vfree(cpuid_entries);
  2030. out:
  2031. return r;
  2032. }
  2033. static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
  2034. struct kvm_cpuid2 *cpuid,
  2035. struct kvm_cpuid_entry2 __user *entries)
  2036. {
  2037. int r;
  2038. r = -E2BIG;
  2039. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  2040. goto out;
  2041. r = -EFAULT;
  2042. if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
  2043. cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
  2044. goto out;
  2045. vcpu->arch.cpuid_nent = cpuid->nent;
  2046. kvm_apic_set_version(vcpu);
  2047. kvm_x86_ops->cpuid_update(vcpu);
  2048. update_cpuid(vcpu);
  2049. return 0;
  2050. out:
  2051. return r;
  2052. }
  2053. static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
  2054. struct kvm_cpuid2 *cpuid,
  2055. struct kvm_cpuid_entry2 __user *entries)
  2056. {
  2057. int r;
  2058. r = -E2BIG;
  2059. if (cpuid->nent < vcpu->arch.cpuid_nent)
  2060. goto out;
  2061. r = -EFAULT;
  2062. if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
  2063. vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
  2064. goto out;
  2065. return 0;
  2066. out:
  2067. cpuid->nent = vcpu->arch.cpuid_nent;
  2068. return r;
  2069. }
  2070. static void cpuid_mask(u32 *word, int wordnum)
  2071. {
  2072. *word &= boot_cpu_data.x86_capability[wordnum];
  2073. }
  2074. static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
  2075. u32 index)
  2076. {
  2077. entry->function = function;
  2078. entry->index = index;
  2079. cpuid_count(entry->function, entry->index,
  2080. &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
  2081. entry->flags = 0;
  2082. }
  2083. static bool supported_xcr0_bit(unsigned bit)
  2084. {
  2085. u64 mask = ((u64)1 << bit);
  2086. return mask & (XSTATE_FP | XSTATE_SSE | XSTATE_YMM) & host_xcr0;
  2087. }
  2088. #define F(x) bit(X86_FEATURE_##x)
  2089. static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
  2090. u32 index, int *nent, int maxnent)
  2091. {
  2092. unsigned f_nx = is_efer_nx() ? F(NX) : 0;
  2093. #ifdef CONFIG_X86_64
  2094. unsigned f_gbpages = (kvm_x86_ops->get_lpage_level() == PT_PDPE_LEVEL)
  2095. ? F(GBPAGES) : 0;
  2096. unsigned f_lm = F(LM);
  2097. #else
  2098. unsigned f_gbpages = 0;
  2099. unsigned f_lm = 0;
  2100. #endif
  2101. unsigned f_rdtscp = kvm_x86_ops->rdtscp_supported() ? F(RDTSCP) : 0;
  2102. /* cpuid 1.edx */
  2103. const u32 kvm_supported_word0_x86_features =
  2104. F(FPU) | F(VME) | F(DE) | F(PSE) |
  2105. F(TSC) | F(MSR) | F(PAE) | F(MCE) |
  2106. F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
  2107. F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
  2108. F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLSH) |
  2109. 0 /* Reserved, DS, ACPI */ | F(MMX) |
  2110. F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
  2111. 0 /* HTT, TM, Reserved, PBE */;
  2112. /* cpuid 0x80000001.edx */
  2113. const u32 kvm_supported_word1_x86_features =
  2114. F(FPU) | F(VME) | F(DE) | F(PSE) |
  2115. F(TSC) | F(MSR) | F(PAE) | F(MCE) |
  2116. F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
  2117. F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
  2118. F(PAT) | F(PSE36) | 0 /* Reserved */ |
  2119. f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
  2120. F(FXSR) | F(FXSR_OPT) | f_gbpages | f_rdtscp |
  2121. 0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW);
  2122. /* cpuid 1.ecx */
  2123. const u32 kvm_supported_word4_x86_features =
  2124. F(XMM3) | F(PCLMULQDQ) | 0 /* DTES64, MONITOR */ |
  2125. 0 /* DS-CPL, VMX, SMX, EST */ |
  2126. 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
  2127. 0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ |
  2128. 0 /* Reserved, DCA */ | F(XMM4_1) |
  2129. F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
  2130. 0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) |
  2131. F(F16C) | F(RDRAND);
  2132. /* cpuid 0x80000001.ecx */
  2133. const u32 kvm_supported_word6_x86_features =
  2134. F(LAHF_LM) | F(CMP_LEGACY) | 0 /*SVM*/ | 0 /* ExtApicSpace */ |
  2135. F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
  2136. F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(XOP) |
  2137. 0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM);
  2138. /* cpuid 0xC0000001.edx */
  2139. const u32 kvm_supported_word5_x86_features =
  2140. F(XSTORE) | F(XSTORE_EN) | F(XCRYPT) | F(XCRYPT_EN) |
  2141. F(ACE2) | F(ACE2_EN) | F(PHE) | F(PHE_EN) |
  2142. F(PMM) | F(PMM_EN);
  2143. /* cpuid 7.0.ebx */
  2144. const u32 kvm_supported_word9_x86_features =
  2145. F(SMEP) | F(FSGSBASE) | F(ERMS);
  2146. /* all calls to cpuid_count() should be made on the same cpu */
  2147. get_cpu();
  2148. do_cpuid_1_ent(entry, function, index);
  2149. ++*nent;
  2150. switch (function) {
  2151. case 0:
  2152. entry->eax = min(entry->eax, (u32)0xd);
  2153. break;
  2154. case 1:
  2155. entry->edx &= kvm_supported_word0_x86_features;
  2156. cpuid_mask(&entry->edx, 0);
  2157. entry->ecx &= kvm_supported_word4_x86_features;
  2158. cpuid_mask(&entry->ecx, 4);
  2159. /* we support x2apic emulation even if host does not support
  2160. * it since we emulate x2apic in software */
  2161. entry->ecx |= F(X2APIC);
  2162. break;
  2163. /* function 2 entries are STATEFUL. That is, repeated cpuid commands
  2164. * may return different values. This forces us to get_cpu() before
  2165. * issuing the first command, and also to emulate this annoying behavior
  2166. * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
  2167. case 2: {
  2168. int t, times = entry->eax & 0xff;
  2169. entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
  2170. entry->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
  2171. for (t = 1; t < times && *nent < maxnent; ++t) {
  2172. do_cpuid_1_ent(&entry[t], function, 0);
  2173. entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
  2174. ++*nent;
  2175. }
  2176. break;
  2177. }
  2178. /* function 4 has additional index. */
  2179. case 4: {
  2180. int i, cache_type;
  2181. entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  2182. /* read more entries until cache_type is zero */
  2183. for (i = 1; *nent < maxnent; ++i) {
  2184. cache_type = entry[i - 1].eax & 0x1f;
  2185. if (!cache_type)
  2186. break;
  2187. do_cpuid_1_ent(&entry[i], function, i);
  2188. entry[i].flags |=
  2189. KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  2190. ++*nent;
  2191. }
  2192. break;
  2193. }
  2194. case 7: {
  2195. entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  2196. /* Mask ebx against host capbability word 9 */
  2197. if (index == 0) {
  2198. entry->ebx &= kvm_supported_word9_x86_features;
  2199. cpuid_mask(&entry->ebx, 9);
  2200. } else
  2201. entry->ebx = 0;
  2202. entry->eax = 0;
  2203. entry->ecx = 0;
  2204. entry->edx = 0;
  2205. break;
  2206. }
  2207. case 9:
  2208. break;
  2209. /* function 0xb has additional index. */
  2210. case 0xb: {
  2211. int i, level_type;
  2212. entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  2213. /* read more entries until level_type is zero */
  2214. for (i = 1; *nent < maxnent; ++i) {
  2215. level_type = entry[i - 1].ecx & 0xff00;
  2216. if (!level_type)
  2217. break;
  2218. do_cpuid_1_ent(&entry[i], function, i);
  2219. entry[i].flags |=
  2220. KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  2221. ++*nent;
  2222. }
  2223. break;
  2224. }
  2225. case 0xd: {
  2226. int idx, i;
  2227. entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  2228. for (idx = 1, i = 1; *nent < maxnent && idx < 64; ++idx) {
  2229. do_cpuid_1_ent(&entry[i], function, idx);
  2230. if (entry[i].eax == 0 || !supported_xcr0_bit(idx))
  2231. continue;
  2232. entry[i].flags |=
  2233. KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  2234. ++*nent;
  2235. ++i;
  2236. }
  2237. break;
  2238. }
  2239. case KVM_CPUID_SIGNATURE: {
  2240. char signature[12] = "KVMKVMKVM\0\0";
  2241. u32 *sigptr = (u32 *)signature;
  2242. entry->eax = 0;
  2243. entry->ebx = sigptr[0];
  2244. entry->ecx = sigptr[1];
  2245. entry->edx = sigptr[2];
  2246. break;
  2247. }
  2248. case KVM_CPUID_FEATURES:
  2249. entry->eax = (1 << KVM_FEATURE_CLOCKSOURCE) |
  2250. (1 << KVM_FEATURE_NOP_IO_DELAY) |
  2251. (1 << KVM_FEATURE_CLOCKSOURCE2) |
  2252. (1 << KVM_FEATURE_ASYNC_PF) |
  2253. (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT);
  2254. if (sched_info_on())
  2255. entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
  2256. entry->ebx = 0;
  2257. entry->ecx = 0;
  2258. entry->edx = 0;
  2259. break;
  2260. case 0x80000000:
  2261. entry->eax = min(entry->eax, 0x8000001a);
  2262. break;
  2263. case 0x80000001:
  2264. entry->edx &= kvm_supported_word1_x86_features;
  2265. cpuid_mask(&entry->edx, 1);
  2266. entry->ecx &= kvm_supported_word6_x86_features;
  2267. cpuid_mask(&entry->ecx, 6);
  2268. break;
  2269. case 0x80000008: {
  2270. unsigned g_phys_as = (entry->eax >> 16) & 0xff;
  2271. unsigned virt_as = max((entry->eax >> 8) & 0xff, 48U);
  2272. unsigned phys_as = entry->eax & 0xff;
  2273. if (!g_phys_as)
  2274. g_phys_as = phys_as;
  2275. entry->eax = g_phys_as | (virt_as << 8);
  2276. entry->ebx = entry->edx = 0;
  2277. break;
  2278. }
  2279. case 0x80000019:
  2280. entry->ecx = entry->edx = 0;
  2281. break;
  2282. case 0x8000001a:
  2283. break;
  2284. case 0x8000001d:
  2285. break;
  2286. /*Add support for Centaur's CPUID instruction*/
  2287. case 0xC0000000:
  2288. /*Just support up to 0xC0000004 now*/
  2289. entry->eax = min(entry->eax, 0xC0000004);
  2290. break;
  2291. case 0xC0000001:
  2292. entry->edx &= kvm_supported_word5_x86_features;
  2293. cpuid_mask(&entry->edx, 5);
  2294. break;
  2295. case 3: /* Processor serial number */
  2296. case 5: /* MONITOR/MWAIT */
  2297. case 6: /* Thermal management */
  2298. case 0xA: /* Architectural Performance Monitoring */
  2299. case 0x80000007: /* Advanced power management */
  2300. case 0xC0000002:
  2301. case 0xC0000003:
  2302. case 0xC0000004:
  2303. default:
  2304. entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
  2305. break;
  2306. }
  2307. kvm_x86_ops->set_supported_cpuid(function, entry);
  2308. put_cpu();
  2309. }
  2310. #undef F
  2311. static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  2312. struct kvm_cpuid_entry2 __user *entries)
  2313. {
  2314. struct kvm_cpuid_entry2 *cpuid_entries;
  2315. int limit, nent = 0, r = -E2BIG;
  2316. u32 func;
  2317. if (cpuid->nent < 1)
  2318. goto out;
  2319. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  2320. cpuid->nent = KVM_MAX_CPUID_ENTRIES;
  2321. r = -ENOMEM;
  2322. cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent);
  2323. if (!cpuid_entries)
  2324. goto out;
  2325. do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent);
  2326. limit = cpuid_entries[0].eax;
  2327. for (func = 1; func <= limit && nent < cpuid->nent; ++func)
  2328. do_cpuid_ent(&cpuid_entries[nent], func, 0,
  2329. &nent, cpuid->nent);
  2330. r = -E2BIG;
  2331. if (nent >= cpuid->nent)
  2332. goto out_free;
  2333. do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent);
  2334. limit = cpuid_entries[nent - 1].eax;
  2335. for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func)
  2336. do_cpuid_ent(&cpuid_entries[nent], func, 0,
  2337. &nent, cpuid->nent);
  2338. r = -E2BIG;
  2339. if (nent >= cpuid->nent)
  2340. goto out_free;
  2341. /* Add support for Centaur's CPUID instruction. */
  2342. if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR) {
  2343. do_cpuid_ent(&cpuid_entries[nent], 0xC0000000, 0,
  2344. &nent, cpuid->nent);
  2345. r = -E2BIG;
  2346. if (nent >= cpuid->nent)
  2347. goto out_free;
  2348. limit = cpuid_entries[nent - 1].eax;
  2349. for (func = 0xC0000001;
  2350. func <= limit && nent < cpuid->nent; ++func)
  2351. do_cpuid_ent(&cpuid_entries[nent], func, 0,
  2352. &nent, cpuid->nent);
  2353. r = -E2BIG;
  2354. if (nent >= cpuid->nent)
  2355. goto out_free;
  2356. }
  2357. do_cpuid_ent(&cpuid_entries[nent], KVM_CPUID_SIGNATURE, 0, &nent,
  2358. cpuid->nent);
  2359. r = -E2BIG;
  2360. if (nent >= cpuid->nent)
  2361. goto out_free;
  2362. do_cpuid_ent(&cpuid_entries[nent], KVM_CPUID_FEATURES, 0, &nent,
  2363. cpuid->nent);
  2364. r = -E2BIG;
  2365. if (nent >= cpuid->nent)
  2366. goto out_free;
  2367. r = -EFAULT;
  2368. if (copy_to_user(entries, cpuid_entries,
  2369. nent * sizeof(struct kvm_cpuid_entry2)))
  2370. goto out_free;
  2371. cpuid->nent = nent;
  2372. r = 0;
  2373. out_free:
  2374. vfree(cpuid_entries);
  2375. out:
  2376. return r;
  2377. }
  2378. static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
  2379. struct kvm_lapic_state *s)
  2380. {
  2381. memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
  2382. return 0;
  2383. }
  2384. static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
  2385. struct kvm_lapic_state *s)
  2386. {
  2387. memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
  2388. kvm_apic_post_state_restore(vcpu);
  2389. update_cr8_intercept(vcpu);
  2390. return 0;
  2391. }
  2392. static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
  2393. struct kvm_interrupt *irq)
  2394. {
  2395. if (irq->irq < 0 || irq->irq >= 256)
  2396. return -EINVAL;
  2397. if (irqchip_in_kernel(vcpu->kvm))
  2398. return -ENXIO;
  2399. kvm_queue_interrupt(vcpu, irq->irq, false);
  2400. kvm_make_request(KVM_REQ_EVENT, vcpu);
  2401. return 0;
  2402. }
  2403. static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
  2404. {
  2405. kvm_inject_nmi(vcpu);
  2406. return 0;
  2407. }
  2408. static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
  2409. struct kvm_tpr_access_ctl *tac)
  2410. {
  2411. if (tac->flags)
  2412. return -EINVAL;
  2413. vcpu->arch.tpr_access_reporting = !!tac->enabled;
  2414. return 0;
  2415. }
  2416. static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
  2417. u64 mcg_cap)
  2418. {
  2419. int r;
  2420. unsigned bank_num = mcg_cap & 0xff, bank;
  2421. r = -EINVAL;
  2422. if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
  2423. goto out;
  2424. if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
  2425. goto out;
  2426. r = 0;
  2427. vcpu->arch.mcg_cap = mcg_cap;
  2428. /* Init IA32_MCG_CTL to all 1s */
  2429. if (mcg_cap & MCG_CTL_P)
  2430. vcpu->arch.mcg_ctl = ~(u64)0;
  2431. /* Init IA32_MCi_CTL to all 1s */
  2432. for (bank = 0; bank < bank_num; bank++)
  2433. vcpu->arch.mce_banks[bank*4] = ~(u64)0;
  2434. out:
  2435. return r;
  2436. }
  2437. static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
  2438. struct kvm_x86_mce *mce)
  2439. {
  2440. u64 mcg_cap = vcpu->arch.mcg_cap;
  2441. unsigned bank_num = mcg_cap & 0xff;
  2442. u64 *banks = vcpu->arch.mce_banks;
  2443. if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
  2444. return -EINVAL;
  2445. /*
  2446. * if IA32_MCG_CTL is not all 1s, the uncorrected error
  2447. * reporting is disabled
  2448. */
  2449. if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
  2450. vcpu->arch.mcg_ctl != ~(u64)0)
  2451. return 0;
  2452. banks += 4 * mce->bank;
  2453. /*
  2454. * if IA32_MCi_CTL is not all 1s, the uncorrected error
  2455. * reporting is disabled for the bank
  2456. */
  2457. if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
  2458. return 0;
  2459. if (mce->status & MCI_STATUS_UC) {
  2460. if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
  2461. !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
  2462. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  2463. return 0;
  2464. }
  2465. if (banks[1] & MCI_STATUS_VAL)
  2466. mce->status |= MCI_STATUS_OVER;
  2467. banks[2] = mce->addr;
  2468. banks[3] = mce->misc;
  2469. vcpu->arch.mcg_status = mce->mcg_status;
  2470. banks[1] = mce->status;
  2471. kvm_queue_exception(vcpu, MC_VECTOR);
  2472. } else if (!(banks[1] & MCI_STATUS_VAL)
  2473. || !(banks[1] & MCI_STATUS_UC)) {
  2474. if (banks[1] & MCI_STATUS_VAL)
  2475. mce->status |= MCI_STATUS_OVER;
  2476. banks[2] = mce->addr;
  2477. banks[3] = mce->misc;
  2478. banks[1] = mce->status;
  2479. } else
  2480. banks[1] |= MCI_STATUS_OVER;
  2481. return 0;
  2482. }
  2483. static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
  2484. struct kvm_vcpu_events *events)
  2485. {
  2486. events->exception.injected =
  2487. vcpu->arch.exception.pending &&
  2488. !kvm_exception_is_soft(vcpu->arch.exception.nr);
  2489. events->exception.nr = vcpu->arch.exception.nr;
  2490. events->exception.has_error_code = vcpu->arch.exception.has_error_code;
  2491. events->exception.pad = 0;
  2492. events->exception.error_code = vcpu->arch.exception.error_code;
  2493. events->interrupt.injected =
  2494. vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft;
  2495. events->interrupt.nr = vcpu->arch.interrupt.nr;
  2496. events->interrupt.soft = 0;
  2497. events->interrupt.shadow =
  2498. kvm_x86_ops->get_interrupt_shadow(vcpu,
  2499. KVM_X86_SHADOW_INT_MOV_SS | KVM_X86_SHADOW_INT_STI);
  2500. events->nmi.injected = vcpu->arch.nmi_injected;
  2501. events->nmi.pending = vcpu->arch.nmi_pending;
  2502. events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
  2503. events->nmi.pad = 0;
  2504. events->sipi_vector = vcpu->arch.sipi_vector;
  2505. events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
  2506. | KVM_VCPUEVENT_VALID_SIPI_VECTOR
  2507. | KVM_VCPUEVENT_VALID_SHADOW);
  2508. memset(&events->reserved, 0, sizeof(events->reserved));
  2509. }
  2510. static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
  2511. struct kvm_vcpu_events *events)
  2512. {
  2513. if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
  2514. | KVM_VCPUEVENT_VALID_SIPI_VECTOR
  2515. | KVM_VCPUEVENT_VALID_SHADOW))
  2516. return -EINVAL;
  2517. vcpu->arch.exception.pending = events->exception.injected;
  2518. vcpu->arch.exception.nr = events->exception.nr;
  2519. vcpu->arch.exception.has_error_code = events->exception.has_error_code;
  2520. vcpu->arch.exception.error_code = events->exception.error_code;
  2521. vcpu->arch.interrupt.pending = events->interrupt.injected;
  2522. vcpu->arch.interrupt.nr = events->interrupt.nr;
  2523. vcpu->arch.interrupt.soft = events->interrupt.soft;
  2524. if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
  2525. kvm_x86_ops->set_interrupt_shadow(vcpu,
  2526. events->interrupt.shadow);
  2527. vcpu->arch.nmi_injected = events->nmi.injected;
  2528. if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
  2529. vcpu->arch.nmi_pending = events->nmi.pending;
  2530. kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);
  2531. if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR)
  2532. vcpu->arch.sipi_vector = events->sipi_vector;
  2533. kvm_make_request(KVM_REQ_EVENT, vcpu);
  2534. return 0;
  2535. }
  2536. static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
  2537. struct kvm_debugregs *dbgregs)
  2538. {
  2539. memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
  2540. dbgregs->dr6 = vcpu->arch.dr6;
  2541. dbgregs->dr7 = vcpu->arch.dr7;
  2542. dbgregs->flags = 0;
  2543. memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
  2544. }
  2545. static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
  2546. struct kvm_debugregs *dbgregs)
  2547. {
  2548. if (dbgregs->flags)
  2549. return -EINVAL;
  2550. memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
  2551. vcpu->arch.dr6 = dbgregs->dr6;
  2552. vcpu->arch.dr7 = dbgregs->dr7;
  2553. return 0;
  2554. }
  2555. static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
  2556. struct kvm_xsave *guest_xsave)
  2557. {
  2558. if (cpu_has_xsave)
  2559. memcpy(guest_xsave->region,
  2560. &vcpu->arch.guest_fpu.state->xsave,
  2561. xstate_size);
  2562. else {
  2563. memcpy(guest_xsave->region,
  2564. &vcpu->arch.guest_fpu.state->fxsave,
  2565. sizeof(struct i387_fxsave_struct));
  2566. *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
  2567. XSTATE_FPSSE;
  2568. }
  2569. }
  2570. static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
  2571. struct kvm_xsave *guest_xsave)
  2572. {
  2573. u64 xstate_bv =
  2574. *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
  2575. if (cpu_has_xsave)
  2576. memcpy(&vcpu->arch.guest_fpu.state->xsave,
  2577. guest_xsave->region, xstate_size);
  2578. else {
  2579. if (xstate_bv & ~XSTATE_FPSSE)
  2580. return -EINVAL;
  2581. memcpy(&vcpu->arch.guest_fpu.state->fxsave,
  2582. guest_xsave->region, sizeof(struct i387_fxsave_struct));
  2583. }
  2584. return 0;
  2585. }
  2586. static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
  2587. struct kvm_xcrs *guest_xcrs)
  2588. {
  2589. if (!cpu_has_xsave) {
  2590. guest_xcrs->nr_xcrs = 0;
  2591. return;
  2592. }
  2593. guest_xcrs->nr_xcrs = 1;
  2594. guest_xcrs->flags = 0;
  2595. guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
  2596. guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
  2597. }
  2598. static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
  2599. struct kvm_xcrs *guest_xcrs)
  2600. {
  2601. int i, r = 0;
  2602. if (!cpu_has_xsave)
  2603. return -EINVAL;
  2604. if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
  2605. return -EINVAL;
  2606. for (i = 0; i < guest_xcrs->nr_xcrs; i++)
  2607. /* Only support XCR0 currently */
  2608. if (guest_xcrs->xcrs[0].xcr == XCR_XFEATURE_ENABLED_MASK) {
  2609. r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
  2610. guest_xcrs->xcrs[0].value);
  2611. break;
  2612. }
  2613. if (r)
  2614. r = -EINVAL;
  2615. return r;
  2616. }
  2617. long kvm_arch_vcpu_ioctl(struct file *filp,
  2618. unsigned int ioctl, unsigned long arg)
  2619. {
  2620. struct kvm_vcpu *vcpu = filp->private_data;
  2621. void __user *argp = (void __user *)arg;
  2622. int r;
  2623. union {
  2624. struct kvm_lapic_state *lapic;
  2625. struct kvm_xsave *xsave;
  2626. struct kvm_xcrs *xcrs;
  2627. void *buffer;
  2628. } u;
  2629. u.buffer = NULL;
  2630. switch (ioctl) {
  2631. case KVM_GET_LAPIC: {
  2632. r = -EINVAL;
  2633. if (!vcpu->arch.apic)
  2634. goto out;
  2635. u.lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
  2636. r = -ENOMEM;
  2637. if (!u.lapic)
  2638. goto out;
  2639. r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
  2640. if (r)
  2641. goto out;
  2642. r = -EFAULT;
  2643. if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
  2644. goto out;
  2645. r = 0;
  2646. break;
  2647. }
  2648. case KVM_SET_LAPIC: {
  2649. r = -EINVAL;
  2650. if (!vcpu->arch.apic)
  2651. goto out;
  2652. u.lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
  2653. r = -ENOMEM;
  2654. if (!u.lapic)
  2655. goto out;
  2656. r = -EFAULT;
  2657. if (copy_from_user(u.lapic, argp, sizeof(struct kvm_lapic_state)))
  2658. goto out;
  2659. r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
  2660. if (r)
  2661. goto out;
  2662. r = 0;
  2663. break;
  2664. }
  2665. case KVM_INTERRUPT: {
  2666. struct kvm_interrupt irq;
  2667. r = -EFAULT;
  2668. if (copy_from_user(&irq, argp, sizeof irq))
  2669. goto out;
  2670. r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
  2671. if (r)
  2672. goto out;
  2673. r = 0;
  2674. break;
  2675. }
  2676. case KVM_NMI: {
  2677. r = kvm_vcpu_ioctl_nmi(vcpu);
  2678. if (r)
  2679. goto out;
  2680. r = 0;
  2681. break;
  2682. }
  2683. case KVM_SET_CPUID: {
  2684. struct kvm_cpuid __user *cpuid_arg = argp;
  2685. struct kvm_cpuid cpuid;
  2686. r = -EFAULT;
  2687. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  2688. goto out;
  2689. r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
  2690. if (r)
  2691. goto out;
  2692. break;
  2693. }
  2694. case KVM_SET_CPUID2: {
  2695. struct kvm_cpuid2 __user *cpuid_arg = argp;
  2696. struct kvm_cpuid2 cpuid;
  2697. r = -EFAULT;
  2698. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  2699. goto out;
  2700. r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
  2701. cpuid_arg->entries);
  2702. if (r)
  2703. goto out;
  2704. break;
  2705. }
  2706. case KVM_GET_CPUID2: {
  2707. struct kvm_cpuid2 __user *cpuid_arg = argp;
  2708. struct kvm_cpuid2 cpuid;
  2709. r = -EFAULT;
  2710. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  2711. goto out;
  2712. r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
  2713. cpuid_arg->entries);
  2714. if (r)
  2715. goto out;
  2716. r = -EFAULT;
  2717. if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
  2718. goto out;
  2719. r = 0;
  2720. break;
  2721. }
  2722. case KVM_GET_MSRS:
  2723. r = msr_io(vcpu, argp, kvm_get_msr, 1);
  2724. break;
  2725. case KVM_SET_MSRS:
  2726. r = msr_io(vcpu, argp, do_set_msr, 0);
  2727. break;
  2728. case KVM_TPR_ACCESS_REPORTING: {
  2729. struct kvm_tpr_access_ctl tac;
  2730. r = -EFAULT;
  2731. if (copy_from_user(&tac, argp, sizeof tac))
  2732. goto out;
  2733. r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
  2734. if (r)
  2735. goto out;
  2736. r = -EFAULT;
  2737. if (copy_to_user(argp, &tac, sizeof tac))
  2738. goto out;
  2739. r = 0;
  2740. break;
  2741. };
  2742. case KVM_SET_VAPIC_ADDR: {
  2743. struct kvm_vapic_addr va;
  2744. r = -EINVAL;
  2745. if (!irqchip_in_kernel(vcpu->kvm))
  2746. goto out;
  2747. r = -EFAULT;
  2748. if (copy_from_user(&va, argp, sizeof va))
  2749. goto out;
  2750. r = 0;
  2751. kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
  2752. break;
  2753. }
  2754. case KVM_X86_SETUP_MCE: {
  2755. u64 mcg_cap;
  2756. r = -EFAULT;
  2757. if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
  2758. goto out;
  2759. r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
  2760. break;
  2761. }
  2762. case KVM_X86_SET_MCE: {
  2763. struct kvm_x86_mce mce;
  2764. r = -EFAULT;
  2765. if (copy_from_user(&mce, argp, sizeof mce))
  2766. goto out;
  2767. r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
  2768. break;
  2769. }
  2770. case KVM_GET_VCPU_EVENTS: {
  2771. struct kvm_vcpu_events events;
  2772. kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
  2773. r = -EFAULT;
  2774. if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
  2775. break;
  2776. r = 0;
  2777. break;
  2778. }
  2779. case KVM_SET_VCPU_EVENTS: {
  2780. struct kvm_vcpu_events events;
  2781. r = -EFAULT;
  2782. if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
  2783. break;
  2784. r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
  2785. break;
  2786. }
  2787. case KVM_GET_DEBUGREGS: {
  2788. struct kvm_debugregs dbgregs;
  2789. kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
  2790. r = -EFAULT;
  2791. if (copy_to_user(argp, &dbgregs,
  2792. sizeof(struct kvm_debugregs)))
  2793. break;
  2794. r = 0;
  2795. break;
  2796. }
  2797. case KVM_SET_DEBUGREGS: {
  2798. struct kvm_debugregs dbgregs;
  2799. r = -EFAULT;
  2800. if (copy_from_user(&dbgregs, argp,
  2801. sizeof(struct kvm_debugregs)))
  2802. break;
  2803. r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
  2804. break;
  2805. }
  2806. case KVM_GET_XSAVE: {
  2807. u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL);
  2808. r = -ENOMEM;
  2809. if (!u.xsave)
  2810. break;
  2811. kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
  2812. r = -EFAULT;
  2813. if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
  2814. break;
  2815. r = 0;
  2816. break;
  2817. }
  2818. case KVM_SET_XSAVE: {
  2819. u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL);
  2820. r = -ENOMEM;
  2821. if (!u.xsave)
  2822. break;
  2823. r = -EFAULT;
  2824. if (copy_from_user(u.xsave, argp, sizeof(struct kvm_xsave)))
  2825. break;
  2826. r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
  2827. break;
  2828. }
  2829. case KVM_GET_XCRS: {
  2830. u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL);
  2831. r = -ENOMEM;
  2832. if (!u.xcrs)
  2833. break;
  2834. kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
  2835. r = -EFAULT;
  2836. if (copy_to_user(argp, u.xcrs,
  2837. sizeof(struct kvm_xcrs)))
  2838. break;
  2839. r = 0;
  2840. break;
  2841. }
  2842. case KVM_SET_XCRS: {
  2843. u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL);
  2844. r = -ENOMEM;
  2845. if (!u.xcrs)
  2846. break;
  2847. r = -EFAULT;
  2848. if (copy_from_user(u.xcrs, argp,
  2849. sizeof(struct kvm_xcrs)))
  2850. break;
  2851. r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
  2852. break;
  2853. }
  2854. case KVM_SET_TSC_KHZ: {
  2855. u32 user_tsc_khz;
  2856. r = -EINVAL;
  2857. if (!kvm_has_tsc_control)
  2858. break;
  2859. user_tsc_khz = (u32)arg;
  2860. if (user_tsc_khz >= kvm_max_guest_tsc_khz)
  2861. goto out;
  2862. kvm_x86_ops->set_tsc_khz(vcpu, user_tsc_khz);
  2863. r = 0;
  2864. goto out;
  2865. }
  2866. case KVM_GET_TSC_KHZ: {
  2867. r = -EIO;
  2868. if (check_tsc_unstable())
  2869. goto out;
  2870. r = vcpu_tsc_khz(vcpu);
  2871. goto out;
  2872. }
  2873. default:
  2874. r = -EINVAL;
  2875. }
  2876. out:
  2877. kfree(u.buffer);
  2878. return r;
  2879. }
  2880. static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
  2881. {
  2882. int ret;
  2883. if (addr > (unsigned int)(-3 * PAGE_SIZE))
  2884. return -1;
  2885. ret = kvm_x86_ops->set_tss_addr(kvm, addr);
  2886. return ret;
  2887. }
  2888. static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
  2889. u64 ident_addr)
  2890. {
  2891. kvm->arch.ept_identity_map_addr = ident_addr;
  2892. return 0;
  2893. }
  2894. static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
  2895. u32 kvm_nr_mmu_pages)
  2896. {
  2897. if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
  2898. return -EINVAL;
  2899. mutex_lock(&kvm->slots_lock);
  2900. spin_lock(&kvm->mmu_lock);
  2901. kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
  2902. kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
  2903. spin_unlock(&kvm->mmu_lock);
  2904. mutex_unlock(&kvm->slots_lock);
  2905. return 0;
  2906. }
  2907. static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
  2908. {
  2909. return kvm->arch.n_max_mmu_pages;
  2910. }
  2911. static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
  2912. {
  2913. int r;
  2914. r = 0;
  2915. switch (chip->chip_id) {
  2916. case KVM_IRQCHIP_PIC_MASTER:
  2917. memcpy(&chip->chip.pic,
  2918. &pic_irqchip(kvm)->pics[0],
  2919. sizeof(struct kvm_pic_state));
  2920. break;
  2921. case KVM_IRQCHIP_PIC_SLAVE:
  2922. memcpy(&chip->chip.pic,
  2923. &pic_irqchip(kvm)->pics[1],
  2924. sizeof(struct kvm_pic_state));
  2925. break;
  2926. case KVM_IRQCHIP_IOAPIC:
  2927. r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
  2928. break;
  2929. default:
  2930. r = -EINVAL;
  2931. break;
  2932. }
  2933. return r;
  2934. }
  2935. static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
  2936. {
  2937. int r;
  2938. r = 0;
  2939. switch (chip->chip_id) {
  2940. case KVM_IRQCHIP_PIC_MASTER:
  2941. spin_lock(&pic_irqchip(kvm)->lock);
  2942. memcpy(&pic_irqchip(kvm)->pics[0],
  2943. &chip->chip.pic,
  2944. sizeof(struct kvm_pic_state));
  2945. spin_unlock(&pic_irqchip(kvm)->lock);
  2946. break;
  2947. case KVM_IRQCHIP_PIC_SLAVE:
  2948. spin_lock(&pic_irqchip(kvm)->lock);
  2949. memcpy(&pic_irqchip(kvm)->pics[1],
  2950. &chip->chip.pic,
  2951. sizeof(struct kvm_pic_state));
  2952. spin_unlock(&pic_irqchip(kvm)->lock);
  2953. break;
  2954. case KVM_IRQCHIP_IOAPIC:
  2955. r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
  2956. break;
  2957. default:
  2958. r = -EINVAL;
  2959. break;
  2960. }
  2961. kvm_pic_update_irq(pic_irqchip(kvm));
  2962. return r;
  2963. }
  2964. static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
  2965. {
  2966. int r = 0;
  2967. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2968. memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
  2969. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2970. return r;
  2971. }
  2972. static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
  2973. {
  2974. int r = 0;
  2975. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2976. memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
  2977. kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
  2978. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2979. return r;
  2980. }
  2981. static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
  2982. {
  2983. int r = 0;
  2984. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2985. memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
  2986. sizeof(ps->channels));
  2987. ps->flags = kvm->arch.vpit->pit_state.flags;
  2988. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2989. memset(&ps->reserved, 0, sizeof(ps->reserved));
  2990. return r;
  2991. }
  2992. static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
  2993. {
  2994. int r = 0, start = 0;
  2995. u32 prev_legacy, cur_legacy;
  2996. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2997. prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
  2998. cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
  2999. if (!prev_legacy && cur_legacy)
  3000. start = 1;
  3001. memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
  3002. sizeof(kvm->arch.vpit->pit_state.channels));
  3003. kvm->arch.vpit->pit_state.flags = ps->flags;
  3004. kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
  3005. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  3006. return r;
  3007. }
  3008. static int kvm_vm_ioctl_reinject(struct kvm *kvm,
  3009. struct kvm_reinject_control *control)
  3010. {
  3011. if (!kvm->arch.vpit)
  3012. return -ENXIO;
  3013. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  3014. kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject;
  3015. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  3016. return 0;
  3017. }
  3018. /*
  3019. * Get (and clear) the dirty memory log for a memory slot.
  3020. */
  3021. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
  3022. struct kvm_dirty_log *log)
  3023. {
  3024. int r, i;
  3025. struct kvm_memory_slot *memslot;
  3026. unsigned long n;
  3027. unsigned long is_dirty = 0;
  3028. mutex_lock(&kvm->slots_lock);
  3029. r = -EINVAL;
  3030. if (log->slot >= KVM_MEMORY_SLOTS)
  3031. goto out;
  3032. memslot = &kvm->memslots->memslots[log->slot];
  3033. r = -ENOENT;
  3034. if (!memslot->dirty_bitmap)
  3035. goto out;
  3036. n = kvm_dirty_bitmap_bytes(memslot);
  3037. for (i = 0; !is_dirty && i < n/sizeof(long); i++)
  3038. is_dirty = memslot->dirty_bitmap[i];
  3039. /* If nothing is dirty, don't bother messing with page tables. */
  3040. if (is_dirty) {
  3041. struct kvm_memslots *slots, *old_slots;
  3042. unsigned long *dirty_bitmap;
  3043. dirty_bitmap = memslot->dirty_bitmap_head;
  3044. if (memslot->dirty_bitmap == dirty_bitmap)
  3045. dirty_bitmap += n / sizeof(long);
  3046. memset(dirty_bitmap, 0, n);
  3047. r = -ENOMEM;
  3048. slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
  3049. if (!slots)
  3050. goto out;
  3051. memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
  3052. slots->memslots[log->slot].dirty_bitmap = dirty_bitmap;
  3053. slots->generation++;
  3054. old_slots = kvm->memslots;
  3055. rcu_assign_pointer(kvm->memslots, slots);
  3056. synchronize_srcu_expedited(&kvm->srcu);
  3057. dirty_bitmap = old_slots->memslots[log->slot].dirty_bitmap;
  3058. kfree(old_slots);
  3059. spin_lock(&kvm->mmu_lock);
  3060. kvm_mmu_slot_remove_write_access(kvm, log->slot);
  3061. spin_unlock(&kvm->mmu_lock);
  3062. r = -EFAULT;
  3063. if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n))
  3064. goto out;
  3065. } else {
  3066. r = -EFAULT;
  3067. if (clear_user(log->dirty_bitmap, n))
  3068. goto out;
  3069. }
  3070. r = 0;
  3071. out:
  3072. mutex_unlock(&kvm->slots_lock);
  3073. return r;
  3074. }
  3075. long kvm_arch_vm_ioctl(struct file *filp,
  3076. unsigned int ioctl, unsigned long arg)
  3077. {
  3078. struct kvm *kvm = filp->private_data;
  3079. void __user *argp = (void __user *)arg;
  3080. int r = -ENOTTY;
  3081. /*
  3082. * This union makes it completely explicit to gcc-3.x
  3083. * that these two variables' stack usage should be
  3084. * combined, not added together.
  3085. */
  3086. union {
  3087. struct kvm_pit_state ps;
  3088. struct kvm_pit_state2 ps2;
  3089. struct kvm_pit_config pit_config;
  3090. } u;
  3091. switch (ioctl) {
  3092. case KVM_SET_TSS_ADDR:
  3093. r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
  3094. if (r < 0)
  3095. goto out;
  3096. break;
  3097. case KVM_SET_IDENTITY_MAP_ADDR: {
  3098. u64 ident_addr;
  3099. r = -EFAULT;
  3100. if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
  3101. goto out;
  3102. r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
  3103. if (r < 0)
  3104. goto out;
  3105. break;
  3106. }
  3107. case KVM_SET_NR_MMU_PAGES:
  3108. r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
  3109. if (r)
  3110. goto out;
  3111. break;
  3112. case KVM_GET_NR_MMU_PAGES:
  3113. r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
  3114. break;
  3115. case KVM_CREATE_IRQCHIP: {
  3116. struct kvm_pic *vpic;
  3117. mutex_lock(&kvm->lock);
  3118. r = -EEXIST;
  3119. if (kvm->arch.vpic)
  3120. goto create_irqchip_unlock;
  3121. r = -ENOMEM;
  3122. vpic = kvm_create_pic(kvm);
  3123. if (vpic) {
  3124. r = kvm_ioapic_init(kvm);
  3125. if (r) {
  3126. mutex_lock(&kvm->slots_lock);
  3127. kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
  3128. &vpic->dev);
  3129. mutex_unlock(&kvm->slots_lock);
  3130. kfree(vpic);
  3131. goto create_irqchip_unlock;
  3132. }
  3133. } else
  3134. goto create_irqchip_unlock;
  3135. smp_wmb();
  3136. kvm->arch.vpic = vpic;
  3137. smp_wmb();
  3138. r = kvm_setup_default_irq_routing(kvm);
  3139. if (r) {
  3140. mutex_lock(&kvm->slots_lock);
  3141. mutex_lock(&kvm->irq_lock);
  3142. kvm_ioapic_destroy(kvm);
  3143. kvm_destroy_pic(kvm);
  3144. mutex_unlock(&kvm->irq_lock);
  3145. mutex_unlock(&kvm->slots_lock);
  3146. }
  3147. create_irqchip_unlock:
  3148. mutex_unlock(&kvm->lock);
  3149. break;
  3150. }
  3151. case KVM_CREATE_PIT:
  3152. u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
  3153. goto create_pit;
  3154. case KVM_CREATE_PIT2:
  3155. r = -EFAULT;
  3156. if (copy_from_user(&u.pit_config, argp,
  3157. sizeof(struct kvm_pit_config)))
  3158. goto out;
  3159. create_pit:
  3160. mutex_lock(&kvm->slots_lock);
  3161. r = -EEXIST;
  3162. if (kvm->arch.vpit)
  3163. goto create_pit_unlock;
  3164. r = -ENOMEM;
  3165. kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
  3166. if (kvm->arch.vpit)
  3167. r = 0;
  3168. create_pit_unlock:
  3169. mutex_unlock(&kvm->slots_lock);
  3170. break;
  3171. case KVM_IRQ_LINE_STATUS:
  3172. case KVM_IRQ_LINE: {
  3173. struct kvm_irq_level irq_event;
  3174. r = -EFAULT;
  3175. if (copy_from_user(&irq_event, argp, sizeof irq_event))
  3176. goto out;
  3177. r = -ENXIO;
  3178. if (irqchip_in_kernel(kvm)) {
  3179. __s32 status;
  3180. status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
  3181. irq_event.irq, irq_event.level);
  3182. if (ioctl == KVM_IRQ_LINE_STATUS) {
  3183. r = -EFAULT;
  3184. irq_event.status = status;
  3185. if (copy_to_user(argp, &irq_event,
  3186. sizeof irq_event))
  3187. goto out;
  3188. }
  3189. r = 0;
  3190. }
  3191. break;
  3192. }
  3193. case KVM_GET_IRQCHIP: {
  3194. /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
  3195. struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
  3196. r = -ENOMEM;
  3197. if (!chip)
  3198. goto out;
  3199. r = -EFAULT;
  3200. if (copy_from_user(chip, argp, sizeof *chip))
  3201. goto get_irqchip_out;
  3202. r = -ENXIO;
  3203. if (!irqchip_in_kernel(kvm))
  3204. goto get_irqchip_out;
  3205. r = kvm_vm_ioctl_get_irqchip(kvm, chip);
  3206. if (r)
  3207. goto get_irqchip_out;
  3208. r = -EFAULT;
  3209. if (copy_to_user(argp, chip, sizeof *chip))
  3210. goto get_irqchip_out;
  3211. r = 0;
  3212. get_irqchip_out:
  3213. kfree(chip);
  3214. if (r)
  3215. goto out;
  3216. break;
  3217. }
  3218. case KVM_SET_IRQCHIP: {
  3219. /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
  3220. struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
  3221. r = -ENOMEM;
  3222. if (!chip)
  3223. goto out;
  3224. r = -EFAULT;
  3225. if (copy_from_user(chip, argp, sizeof *chip))
  3226. goto set_irqchip_out;
  3227. r = -ENXIO;
  3228. if (!irqchip_in_kernel(kvm))
  3229. goto set_irqchip_out;
  3230. r = kvm_vm_ioctl_set_irqchip(kvm, chip);
  3231. if (r)
  3232. goto set_irqchip_out;
  3233. r = 0;
  3234. set_irqchip_out:
  3235. kfree(chip);
  3236. if (r)
  3237. goto out;
  3238. break;
  3239. }
  3240. case KVM_GET_PIT: {
  3241. r = -EFAULT;
  3242. if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
  3243. goto out;
  3244. r = -ENXIO;
  3245. if (!kvm->arch.vpit)
  3246. goto out;
  3247. r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
  3248. if (r)
  3249. goto out;
  3250. r = -EFAULT;
  3251. if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
  3252. goto out;
  3253. r = 0;
  3254. break;
  3255. }
  3256. case KVM_SET_PIT: {
  3257. r = -EFAULT;
  3258. if (copy_from_user(&u.ps, argp, sizeof u.ps))
  3259. goto out;
  3260. r = -ENXIO;
  3261. if (!kvm->arch.vpit)
  3262. goto out;
  3263. r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
  3264. if (r)
  3265. goto out;
  3266. r = 0;
  3267. break;
  3268. }
  3269. case KVM_GET_PIT2: {
  3270. r = -ENXIO;
  3271. if (!kvm->arch.vpit)
  3272. goto out;
  3273. r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
  3274. if (r)
  3275. goto out;
  3276. r = -EFAULT;
  3277. if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
  3278. goto out;
  3279. r = 0;
  3280. break;
  3281. }
  3282. case KVM_SET_PIT2: {
  3283. r = -EFAULT;
  3284. if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
  3285. goto out;
  3286. r = -ENXIO;
  3287. if (!kvm->arch.vpit)
  3288. goto out;
  3289. r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
  3290. if (r)
  3291. goto out;
  3292. r = 0;
  3293. break;
  3294. }
  3295. case KVM_REINJECT_CONTROL: {
  3296. struct kvm_reinject_control control;
  3297. r = -EFAULT;
  3298. if (copy_from_user(&control, argp, sizeof(control)))
  3299. goto out;
  3300. r = kvm_vm_ioctl_reinject(kvm, &control);
  3301. if (r)
  3302. goto out;
  3303. r = 0;
  3304. break;
  3305. }
  3306. case KVM_XEN_HVM_CONFIG: {
  3307. r = -EFAULT;
  3308. if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
  3309. sizeof(struct kvm_xen_hvm_config)))
  3310. goto out;
  3311. r = -EINVAL;
  3312. if (kvm->arch.xen_hvm_config.flags)
  3313. goto out;
  3314. r = 0;
  3315. break;
  3316. }
  3317. case KVM_SET_CLOCK: {
  3318. struct kvm_clock_data user_ns;
  3319. u64 now_ns;
  3320. s64 delta;
  3321. r = -EFAULT;
  3322. if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
  3323. goto out;
  3324. r = -EINVAL;
  3325. if (user_ns.flags)
  3326. goto out;
  3327. r = 0;
  3328. local_irq_disable();
  3329. now_ns = get_kernel_ns();
  3330. delta = user_ns.clock - now_ns;
  3331. local_irq_enable();
  3332. kvm->arch.kvmclock_offset = delta;
  3333. break;
  3334. }
  3335. case KVM_GET_CLOCK: {
  3336. struct kvm_clock_data user_ns;
  3337. u64 now_ns;
  3338. local_irq_disable();
  3339. now_ns = get_kernel_ns();
  3340. user_ns.clock = kvm->arch.kvmclock_offset + now_ns;
  3341. local_irq_enable();
  3342. user_ns.flags = 0;
  3343. memset(&user_ns.pad, 0, sizeof(user_ns.pad));
  3344. r = -EFAULT;
  3345. if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
  3346. goto out;
  3347. r = 0;
  3348. break;
  3349. }
  3350. default:
  3351. ;
  3352. }
  3353. out:
  3354. return r;
  3355. }
  3356. static void kvm_init_msr_list(void)
  3357. {
  3358. u32 dummy[2];
  3359. unsigned i, j;
  3360. /* skip the first msrs in the list. KVM-specific */
  3361. for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) {
  3362. if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
  3363. continue;
  3364. if (j < i)
  3365. msrs_to_save[j] = msrs_to_save[i];
  3366. j++;
  3367. }
  3368. num_msrs_to_save = j;
  3369. }
  3370. static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
  3371. const void *v)
  3372. {
  3373. int handled = 0;
  3374. int n;
  3375. do {
  3376. n = min(len, 8);
  3377. if (!(vcpu->arch.apic &&
  3378. !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, n, v))
  3379. && kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
  3380. break;
  3381. handled += n;
  3382. addr += n;
  3383. len -= n;
  3384. v += n;
  3385. } while (len);
  3386. return handled;
  3387. }
  3388. static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
  3389. {
  3390. int handled = 0;
  3391. int n;
  3392. do {
  3393. n = min(len, 8);
  3394. if (!(vcpu->arch.apic &&
  3395. !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, n, v))
  3396. && kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
  3397. break;
  3398. trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v);
  3399. handled += n;
  3400. addr += n;
  3401. len -= n;
  3402. v += n;
  3403. } while (len);
  3404. return handled;
  3405. }
  3406. static void kvm_set_segment(struct kvm_vcpu *vcpu,
  3407. struct kvm_segment *var, int seg)
  3408. {
  3409. kvm_x86_ops->set_segment(vcpu, var, seg);
  3410. }
  3411. void kvm_get_segment(struct kvm_vcpu *vcpu,
  3412. struct kvm_segment *var, int seg)
  3413. {
  3414. kvm_x86_ops->get_segment(vcpu, var, seg);
  3415. }
  3416. static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
  3417. {
  3418. return gpa;
  3419. }
  3420. static gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
  3421. {
  3422. gpa_t t_gpa;
  3423. struct x86_exception exception;
  3424. BUG_ON(!mmu_is_nested(vcpu));
  3425. /* NPT walks are always user-walks */
  3426. access |= PFERR_USER_MASK;
  3427. t_gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &exception);
  3428. return t_gpa;
  3429. }
  3430. gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
  3431. struct x86_exception *exception)
  3432. {
  3433. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3434. return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
  3435. }
  3436. gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
  3437. struct x86_exception *exception)
  3438. {
  3439. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3440. access |= PFERR_FETCH_MASK;
  3441. return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
  3442. }
  3443. gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
  3444. struct x86_exception *exception)
  3445. {
  3446. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3447. access |= PFERR_WRITE_MASK;
  3448. return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
  3449. }
  3450. /* uses this to access any guest's mapped memory without checking CPL */
  3451. gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
  3452. struct x86_exception *exception)
  3453. {
  3454. return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
  3455. }
  3456. static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
  3457. struct kvm_vcpu *vcpu, u32 access,
  3458. struct x86_exception *exception)
  3459. {
  3460. void *data = val;
  3461. int r = X86EMUL_CONTINUE;
  3462. while (bytes) {
  3463. gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
  3464. exception);
  3465. unsigned offset = addr & (PAGE_SIZE-1);
  3466. unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
  3467. int ret;
  3468. if (gpa == UNMAPPED_GVA)
  3469. return X86EMUL_PROPAGATE_FAULT;
  3470. ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
  3471. if (ret < 0) {
  3472. r = X86EMUL_IO_NEEDED;
  3473. goto out;
  3474. }
  3475. bytes -= toread;
  3476. data += toread;
  3477. addr += toread;
  3478. }
  3479. out:
  3480. return r;
  3481. }
  3482. /* used for instruction fetching */
  3483. static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
  3484. gva_t addr, void *val, unsigned int bytes,
  3485. struct x86_exception *exception)
  3486. {
  3487. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3488. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3489. return kvm_read_guest_virt_helper(addr, val, bytes, vcpu,
  3490. access | PFERR_FETCH_MASK,
  3491. exception);
  3492. }
  3493. int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
  3494. gva_t addr, void *val, unsigned int bytes,
  3495. struct x86_exception *exception)
  3496. {
  3497. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3498. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3499. return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
  3500. exception);
  3501. }
  3502. EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
  3503. static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt,
  3504. gva_t addr, void *val, unsigned int bytes,
  3505. struct x86_exception *exception)
  3506. {
  3507. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3508. return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception);
  3509. }
  3510. int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
  3511. gva_t addr, void *val,
  3512. unsigned int bytes,
  3513. struct x86_exception *exception)
  3514. {
  3515. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3516. void *data = val;
  3517. int r = X86EMUL_CONTINUE;
  3518. while (bytes) {
  3519. gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
  3520. PFERR_WRITE_MASK,
  3521. exception);
  3522. unsigned offset = addr & (PAGE_SIZE-1);
  3523. unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
  3524. int ret;
  3525. if (gpa == UNMAPPED_GVA)
  3526. return X86EMUL_PROPAGATE_FAULT;
  3527. ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
  3528. if (ret < 0) {
  3529. r = X86EMUL_IO_NEEDED;
  3530. goto out;
  3531. }
  3532. bytes -= towrite;
  3533. data += towrite;
  3534. addr += towrite;
  3535. }
  3536. out:
  3537. return r;
  3538. }
  3539. EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
  3540. static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
  3541. gpa_t *gpa, struct x86_exception *exception,
  3542. bool write)
  3543. {
  3544. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3545. if (vcpu_match_mmio_gva(vcpu, gva) &&
  3546. check_write_user_access(vcpu, write, access,
  3547. vcpu->arch.access)) {
  3548. *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
  3549. (gva & (PAGE_SIZE - 1));
  3550. trace_vcpu_match_mmio(gva, *gpa, write, false);
  3551. return 1;
  3552. }
  3553. if (write)
  3554. access |= PFERR_WRITE_MASK;
  3555. *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
  3556. if (*gpa == UNMAPPED_GVA)
  3557. return -1;
  3558. /* For APIC access vmexit */
  3559. if ((*gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  3560. return 1;
  3561. if (vcpu_match_mmio_gpa(vcpu, *gpa)) {
  3562. trace_vcpu_match_mmio(gva, *gpa, write, true);
  3563. return 1;
  3564. }
  3565. return 0;
  3566. }
  3567. int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
  3568. const void *val, int bytes)
  3569. {
  3570. int ret;
  3571. ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
  3572. if (ret < 0)
  3573. return 0;
  3574. kvm_mmu_pte_write(vcpu, gpa, val, bytes, 1);
  3575. return 1;
  3576. }
  3577. struct read_write_emulator_ops {
  3578. int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
  3579. int bytes);
  3580. int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
  3581. void *val, int bytes);
  3582. int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
  3583. int bytes, void *val);
  3584. int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
  3585. void *val, int bytes);
  3586. bool write;
  3587. };
  3588. static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
  3589. {
  3590. if (vcpu->mmio_read_completed) {
  3591. memcpy(val, vcpu->mmio_data, bytes);
  3592. trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
  3593. vcpu->mmio_phys_addr, *(u64 *)val);
  3594. vcpu->mmio_read_completed = 0;
  3595. return 1;
  3596. }
  3597. return 0;
  3598. }
  3599. static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
  3600. void *val, int bytes)
  3601. {
  3602. return !kvm_read_guest(vcpu->kvm, gpa, val, bytes);
  3603. }
  3604. static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
  3605. void *val, int bytes)
  3606. {
  3607. return emulator_write_phys(vcpu, gpa, val, bytes);
  3608. }
  3609. static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
  3610. {
  3611. trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
  3612. return vcpu_mmio_write(vcpu, gpa, bytes, val);
  3613. }
  3614. static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
  3615. void *val, int bytes)
  3616. {
  3617. trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
  3618. return X86EMUL_IO_NEEDED;
  3619. }
  3620. static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
  3621. void *val, int bytes)
  3622. {
  3623. memcpy(vcpu->mmio_data, val, bytes);
  3624. memcpy(vcpu->run->mmio.data, vcpu->mmio_data, 8);
  3625. return X86EMUL_CONTINUE;
  3626. }
  3627. static struct read_write_emulator_ops read_emultor = {
  3628. .read_write_prepare = read_prepare,
  3629. .read_write_emulate = read_emulate,
  3630. .read_write_mmio = vcpu_mmio_read,
  3631. .read_write_exit_mmio = read_exit_mmio,
  3632. };
  3633. static struct read_write_emulator_ops write_emultor = {
  3634. .read_write_emulate = write_emulate,
  3635. .read_write_mmio = write_mmio,
  3636. .read_write_exit_mmio = write_exit_mmio,
  3637. .write = true,
  3638. };
  3639. static int emulator_read_write_onepage(unsigned long addr, void *val,
  3640. unsigned int bytes,
  3641. struct x86_exception *exception,
  3642. struct kvm_vcpu *vcpu,
  3643. struct read_write_emulator_ops *ops)
  3644. {
  3645. gpa_t gpa;
  3646. int handled, ret;
  3647. bool write = ops->write;
  3648. if (ops->read_write_prepare &&
  3649. ops->read_write_prepare(vcpu, val, bytes))
  3650. return X86EMUL_CONTINUE;
  3651. ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
  3652. if (ret < 0)
  3653. return X86EMUL_PROPAGATE_FAULT;
  3654. /* For APIC access vmexit */
  3655. if (ret)
  3656. goto mmio;
  3657. if (ops->read_write_emulate(vcpu, gpa, val, bytes))
  3658. return X86EMUL_CONTINUE;
  3659. mmio:
  3660. /*
  3661. * Is this MMIO handled locally?
  3662. */
  3663. handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
  3664. if (handled == bytes)
  3665. return X86EMUL_CONTINUE;
  3666. gpa += handled;
  3667. bytes -= handled;
  3668. val += handled;
  3669. vcpu->mmio_needed = 1;
  3670. vcpu->run->exit_reason = KVM_EXIT_MMIO;
  3671. vcpu->run->mmio.phys_addr = vcpu->mmio_phys_addr = gpa;
  3672. vcpu->mmio_size = bytes;
  3673. vcpu->run->mmio.len = min(vcpu->mmio_size, 8);
  3674. vcpu->run->mmio.is_write = vcpu->mmio_is_write = write;
  3675. vcpu->mmio_index = 0;
  3676. return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
  3677. }
  3678. int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr,
  3679. void *val, unsigned int bytes,
  3680. struct x86_exception *exception,
  3681. struct read_write_emulator_ops *ops)
  3682. {
  3683. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3684. /* Crossing a page boundary? */
  3685. if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
  3686. int rc, now;
  3687. now = -addr & ~PAGE_MASK;
  3688. rc = emulator_read_write_onepage(addr, val, now, exception,
  3689. vcpu, ops);
  3690. if (rc != X86EMUL_CONTINUE)
  3691. return rc;
  3692. addr += now;
  3693. val += now;
  3694. bytes -= now;
  3695. }
  3696. return emulator_read_write_onepage(addr, val, bytes, exception,
  3697. vcpu, ops);
  3698. }
  3699. static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
  3700. unsigned long addr,
  3701. void *val,
  3702. unsigned int bytes,
  3703. struct x86_exception *exception)
  3704. {
  3705. return emulator_read_write(ctxt, addr, val, bytes,
  3706. exception, &read_emultor);
  3707. }
  3708. int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
  3709. unsigned long addr,
  3710. const void *val,
  3711. unsigned int bytes,
  3712. struct x86_exception *exception)
  3713. {
  3714. return emulator_read_write(ctxt, addr, (void *)val, bytes,
  3715. exception, &write_emultor);
  3716. }
  3717. #define CMPXCHG_TYPE(t, ptr, old, new) \
  3718. (cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))
  3719. #ifdef CONFIG_X86_64
  3720. # define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
  3721. #else
  3722. # define CMPXCHG64(ptr, old, new) \
  3723. (cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
  3724. #endif
  3725. static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
  3726. unsigned long addr,
  3727. const void *old,
  3728. const void *new,
  3729. unsigned int bytes,
  3730. struct x86_exception *exception)
  3731. {
  3732. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3733. gpa_t gpa;
  3734. struct page *page;
  3735. char *kaddr;
  3736. bool exchanged;
  3737. /* guests cmpxchg8b have to be emulated atomically */
  3738. if (bytes > 8 || (bytes & (bytes - 1)))
  3739. goto emul_write;
  3740. gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
  3741. if (gpa == UNMAPPED_GVA ||
  3742. (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  3743. goto emul_write;
  3744. if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
  3745. goto emul_write;
  3746. page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  3747. if (is_error_page(page)) {
  3748. kvm_release_page_clean(page);
  3749. goto emul_write;
  3750. }
  3751. kaddr = kmap_atomic(page, KM_USER0);
  3752. kaddr += offset_in_page(gpa);
  3753. switch (bytes) {
  3754. case 1:
  3755. exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
  3756. break;
  3757. case 2:
  3758. exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
  3759. break;
  3760. case 4:
  3761. exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
  3762. break;
  3763. case 8:
  3764. exchanged = CMPXCHG64(kaddr, old, new);
  3765. break;
  3766. default:
  3767. BUG();
  3768. }
  3769. kunmap_atomic(kaddr, KM_USER0);
  3770. kvm_release_page_dirty(page);
  3771. if (!exchanged)
  3772. return X86EMUL_CMPXCHG_FAILED;
  3773. kvm_mmu_pte_write(vcpu, gpa, new, bytes, 1);
  3774. return X86EMUL_CONTINUE;
  3775. emul_write:
  3776. printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
  3777. return emulator_write_emulated(ctxt, addr, new, bytes, exception);
  3778. }
  3779. static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
  3780. {
  3781. /* TODO: String I/O for in kernel device */
  3782. int r;
  3783. if (vcpu->arch.pio.in)
  3784. r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
  3785. vcpu->arch.pio.size, pd);
  3786. else
  3787. r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
  3788. vcpu->arch.pio.port, vcpu->arch.pio.size,
  3789. pd);
  3790. return r;
  3791. }
  3792. static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
  3793. int size, unsigned short port, void *val,
  3794. unsigned int count)
  3795. {
  3796. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3797. if (vcpu->arch.pio.count)
  3798. goto data_avail;
  3799. trace_kvm_pio(0, port, size, count);
  3800. vcpu->arch.pio.port = port;
  3801. vcpu->arch.pio.in = 1;
  3802. vcpu->arch.pio.count = count;
  3803. vcpu->arch.pio.size = size;
  3804. if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
  3805. data_avail:
  3806. memcpy(val, vcpu->arch.pio_data, size * count);
  3807. vcpu->arch.pio.count = 0;
  3808. return 1;
  3809. }
  3810. vcpu->run->exit_reason = KVM_EXIT_IO;
  3811. vcpu->run->io.direction = KVM_EXIT_IO_IN;
  3812. vcpu->run->io.size = size;
  3813. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  3814. vcpu->run->io.count = count;
  3815. vcpu->run->io.port = port;
  3816. return 0;
  3817. }
  3818. static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
  3819. int size, unsigned short port,
  3820. const void *val, unsigned int count)
  3821. {
  3822. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3823. trace_kvm_pio(1, port, size, count);
  3824. vcpu->arch.pio.port = port;
  3825. vcpu->arch.pio.in = 0;
  3826. vcpu->arch.pio.count = count;
  3827. vcpu->arch.pio.size = size;
  3828. memcpy(vcpu->arch.pio_data, val, size * count);
  3829. if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
  3830. vcpu->arch.pio.count = 0;
  3831. return 1;
  3832. }
  3833. vcpu->run->exit_reason = KVM_EXIT_IO;
  3834. vcpu->run->io.direction = KVM_EXIT_IO_OUT;
  3835. vcpu->run->io.size = size;
  3836. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  3837. vcpu->run->io.count = count;
  3838. vcpu->run->io.port = port;
  3839. return 0;
  3840. }
  3841. static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
  3842. {
  3843. return kvm_x86_ops->get_segment_base(vcpu, seg);
  3844. }
  3845. static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
  3846. {
  3847. kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
  3848. }
  3849. int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
  3850. {
  3851. if (!need_emulate_wbinvd(vcpu))
  3852. return X86EMUL_CONTINUE;
  3853. if (kvm_x86_ops->has_wbinvd_exit()) {
  3854. int cpu = get_cpu();
  3855. cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
  3856. smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
  3857. wbinvd_ipi, NULL, 1);
  3858. put_cpu();
  3859. cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
  3860. } else
  3861. wbinvd();
  3862. return X86EMUL_CONTINUE;
  3863. }
  3864. EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
  3865. static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
  3866. {
  3867. kvm_emulate_wbinvd(emul_to_vcpu(ctxt));
  3868. }
  3869. int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
  3870. {
  3871. return _kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
  3872. }
  3873. int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
  3874. {
  3875. return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
  3876. }
  3877. static u64 mk_cr_64(u64 curr_cr, u32 new_val)
  3878. {
  3879. return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
  3880. }
  3881. static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
  3882. {
  3883. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3884. unsigned long value;
  3885. switch (cr) {
  3886. case 0:
  3887. value = kvm_read_cr0(vcpu);
  3888. break;
  3889. case 2:
  3890. value = vcpu->arch.cr2;
  3891. break;
  3892. case 3:
  3893. value = kvm_read_cr3(vcpu);
  3894. break;
  3895. case 4:
  3896. value = kvm_read_cr4(vcpu);
  3897. break;
  3898. case 8:
  3899. value = kvm_get_cr8(vcpu);
  3900. break;
  3901. default:
  3902. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
  3903. return 0;
  3904. }
  3905. return value;
  3906. }
  3907. static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
  3908. {
  3909. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3910. int res = 0;
  3911. switch (cr) {
  3912. case 0:
  3913. res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
  3914. break;
  3915. case 2:
  3916. vcpu->arch.cr2 = val;
  3917. break;
  3918. case 3:
  3919. res = kvm_set_cr3(vcpu, val);
  3920. break;
  3921. case 4:
  3922. res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
  3923. break;
  3924. case 8:
  3925. res = kvm_set_cr8(vcpu, val);
  3926. break;
  3927. default:
  3928. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
  3929. res = -1;
  3930. }
  3931. return res;
  3932. }
  3933. static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
  3934. {
  3935. return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
  3936. }
  3937. static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
  3938. {
  3939. kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
  3940. }
  3941. static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
  3942. {
  3943. kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
  3944. }
  3945. static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
  3946. {
  3947. kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
  3948. }
  3949. static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
  3950. {
  3951. kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
  3952. }
  3953. static unsigned long emulator_get_cached_segment_base(
  3954. struct x86_emulate_ctxt *ctxt, int seg)
  3955. {
  3956. return get_segment_base(emul_to_vcpu(ctxt), seg);
  3957. }
  3958. static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
  3959. struct desc_struct *desc, u32 *base3,
  3960. int seg)
  3961. {
  3962. struct kvm_segment var;
  3963. kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
  3964. *selector = var.selector;
  3965. if (var.unusable)
  3966. return false;
  3967. if (var.g)
  3968. var.limit >>= 12;
  3969. set_desc_limit(desc, var.limit);
  3970. set_desc_base(desc, (unsigned long)var.base);
  3971. #ifdef CONFIG_X86_64
  3972. if (base3)
  3973. *base3 = var.base >> 32;
  3974. #endif
  3975. desc->type = var.type;
  3976. desc->s = var.s;
  3977. desc->dpl = var.dpl;
  3978. desc->p = var.present;
  3979. desc->avl = var.avl;
  3980. desc->l = var.l;
  3981. desc->d = var.db;
  3982. desc->g = var.g;
  3983. return true;
  3984. }
  3985. static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
  3986. struct desc_struct *desc, u32 base3,
  3987. int seg)
  3988. {
  3989. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3990. struct kvm_segment var;
  3991. var.selector = selector;
  3992. var.base = get_desc_base(desc);
  3993. #ifdef CONFIG_X86_64
  3994. var.base |= ((u64)base3) << 32;
  3995. #endif
  3996. var.limit = get_desc_limit(desc);
  3997. if (desc->g)
  3998. var.limit = (var.limit << 12) | 0xfff;
  3999. var.type = desc->type;
  4000. var.present = desc->p;
  4001. var.dpl = desc->dpl;
  4002. var.db = desc->d;
  4003. var.s = desc->s;
  4004. var.l = desc->l;
  4005. var.g = desc->g;
  4006. var.avl = desc->avl;
  4007. var.present = desc->p;
  4008. var.unusable = !var.present;
  4009. var.padding = 0;
  4010. kvm_set_segment(vcpu, &var, seg);
  4011. return;
  4012. }
  4013. static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
  4014. u32 msr_index, u64 *pdata)
  4015. {
  4016. return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
  4017. }
  4018. static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
  4019. u32 msr_index, u64 data)
  4020. {
  4021. return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
  4022. }
  4023. static void emulator_halt(struct x86_emulate_ctxt *ctxt)
  4024. {
  4025. emul_to_vcpu(ctxt)->arch.halt_request = 1;
  4026. }
  4027. static void emulator_get_fpu(struct x86_emulate_ctxt *ctxt)
  4028. {
  4029. preempt_disable();
  4030. kvm_load_guest_fpu(emul_to_vcpu(ctxt));
  4031. /*
  4032. * CR0.TS may reference the host fpu state, not the guest fpu state,
  4033. * so it may be clear at this point.
  4034. */
  4035. clts();
  4036. }
  4037. static void emulator_put_fpu(struct x86_emulate_ctxt *ctxt)
  4038. {
  4039. preempt_enable();
  4040. }
  4041. static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
  4042. struct x86_instruction_info *info,
  4043. enum x86_intercept_stage stage)
  4044. {
  4045. return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
  4046. }
  4047. static struct x86_emulate_ops emulate_ops = {
  4048. .read_std = kvm_read_guest_virt_system,
  4049. .write_std = kvm_write_guest_virt_system,
  4050. .fetch = kvm_fetch_guest_virt,
  4051. .read_emulated = emulator_read_emulated,
  4052. .write_emulated = emulator_write_emulated,
  4053. .cmpxchg_emulated = emulator_cmpxchg_emulated,
  4054. .invlpg = emulator_invlpg,
  4055. .pio_in_emulated = emulator_pio_in_emulated,
  4056. .pio_out_emulated = emulator_pio_out_emulated,
  4057. .get_segment = emulator_get_segment,
  4058. .set_segment = emulator_set_segment,
  4059. .get_cached_segment_base = emulator_get_cached_segment_base,
  4060. .get_gdt = emulator_get_gdt,
  4061. .get_idt = emulator_get_idt,
  4062. .set_gdt = emulator_set_gdt,
  4063. .set_idt = emulator_set_idt,
  4064. .get_cr = emulator_get_cr,
  4065. .set_cr = emulator_set_cr,
  4066. .cpl = emulator_get_cpl,
  4067. .get_dr = emulator_get_dr,
  4068. .set_dr = emulator_set_dr,
  4069. .set_msr = emulator_set_msr,
  4070. .get_msr = emulator_get_msr,
  4071. .halt = emulator_halt,
  4072. .wbinvd = emulator_wbinvd,
  4073. .fix_hypercall = emulator_fix_hypercall,
  4074. .get_fpu = emulator_get_fpu,
  4075. .put_fpu = emulator_put_fpu,
  4076. .intercept = emulator_intercept,
  4077. };
  4078. static void cache_all_regs(struct kvm_vcpu *vcpu)
  4079. {
  4080. kvm_register_read(vcpu, VCPU_REGS_RAX);
  4081. kvm_register_read(vcpu, VCPU_REGS_RSP);
  4082. kvm_register_read(vcpu, VCPU_REGS_RIP);
  4083. vcpu->arch.regs_dirty = ~0;
  4084. }
  4085. static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
  4086. {
  4087. u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask);
  4088. /*
  4089. * an sti; sti; sequence only disable interrupts for the first
  4090. * instruction. So, if the last instruction, be it emulated or
  4091. * not, left the system with the INT_STI flag enabled, it
  4092. * means that the last instruction is an sti. We should not
  4093. * leave the flag on in this case. The same goes for mov ss
  4094. */
  4095. if (!(int_shadow & mask))
  4096. kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
  4097. }
  4098. static void inject_emulated_exception(struct kvm_vcpu *vcpu)
  4099. {
  4100. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  4101. if (ctxt->exception.vector == PF_VECTOR)
  4102. kvm_propagate_fault(vcpu, &ctxt->exception);
  4103. else if (ctxt->exception.error_code_valid)
  4104. kvm_queue_exception_e(vcpu, ctxt->exception.vector,
  4105. ctxt->exception.error_code);
  4106. else
  4107. kvm_queue_exception(vcpu, ctxt->exception.vector);
  4108. }
  4109. static void init_decode_cache(struct x86_emulate_ctxt *ctxt,
  4110. const unsigned long *regs)
  4111. {
  4112. memset(&ctxt->twobyte, 0,
  4113. (void *)&ctxt->regs - (void *)&ctxt->twobyte);
  4114. memcpy(ctxt->regs, regs, sizeof(ctxt->regs));
  4115. ctxt->fetch.start = 0;
  4116. ctxt->fetch.end = 0;
  4117. ctxt->io_read.pos = 0;
  4118. ctxt->io_read.end = 0;
  4119. ctxt->mem_read.pos = 0;
  4120. ctxt->mem_read.end = 0;
  4121. }
  4122. static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
  4123. {
  4124. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  4125. int cs_db, cs_l;
  4126. /*
  4127. * TODO: fix emulate.c to use guest_read/write_register
  4128. * instead of direct ->regs accesses, can save hundred cycles
  4129. * on Intel for instructions that don't read/change RSP, for
  4130. * for example.
  4131. */
  4132. cache_all_regs(vcpu);
  4133. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  4134. ctxt->eflags = kvm_get_rflags(vcpu);
  4135. ctxt->eip = kvm_rip_read(vcpu);
  4136. ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL :
  4137. (ctxt->eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 :
  4138. cs_l ? X86EMUL_MODE_PROT64 :
  4139. cs_db ? X86EMUL_MODE_PROT32 :
  4140. X86EMUL_MODE_PROT16;
  4141. ctxt->guest_mode = is_guest_mode(vcpu);
  4142. init_decode_cache(ctxt, vcpu->arch.regs);
  4143. vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
  4144. }
  4145. int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
  4146. {
  4147. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  4148. int ret;
  4149. init_emulate_ctxt(vcpu);
  4150. ctxt->op_bytes = 2;
  4151. ctxt->ad_bytes = 2;
  4152. ctxt->_eip = ctxt->eip + inc_eip;
  4153. ret = emulate_int_real(ctxt, irq);
  4154. if (ret != X86EMUL_CONTINUE)
  4155. return EMULATE_FAIL;
  4156. ctxt->eip = ctxt->_eip;
  4157. memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
  4158. kvm_rip_write(vcpu, ctxt->eip);
  4159. kvm_set_rflags(vcpu, ctxt->eflags);
  4160. if (irq == NMI_VECTOR)
  4161. vcpu->arch.nmi_pending = false;
  4162. else
  4163. vcpu->arch.interrupt.pending = false;
  4164. return EMULATE_DONE;
  4165. }
  4166. EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);
  4167. static int handle_emulation_failure(struct kvm_vcpu *vcpu)
  4168. {
  4169. int r = EMULATE_DONE;
  4170. ++vcpu->stat.insn_emulation_fail;
  4171. trace_kvm_emulate_insn_failed(vcpu);
  4172. if (!is_guest_mode(vcpu)) {
  4173. vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  4174. vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
  4175. vcpu->run->internal.ndata = 0;
  4176. r = EMULATE_FAIL;
  4177. }
  4178. kvm_queue_exception(vcpu, UD_VECTOR);
  4179. return r;
  4180. }
  4181. static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
  4182. {
  4183. gpa_t gpa;
  4184. if (tdp_enabled)
  4185. return false;
  4186. /*
  4187. * if emulation was due to access to shadowed page table
  4188. * and it failed try to unshadow page and re-entetr the
  4189. * guest to let CPU execute the instruction.
  4190. */
  4191. if (kvm_mmu_unprotect_page_virt(vcpu, gva))
  4192. return true;
  4193. gpa = kvm_mmu_gva_to_gpa_system(vcpu, gva, NULL);
  4194. if (gpa == UNMAPPED_GVA)
  4195. return true; /* let cpu generate fault */
  4196. if (!kvm_is_error_hva(gfn_to_hva(vcpu->kvm, gpa >> PAGE_SHIFT)))
  4197. return true;
  4198. return false;
  4199. }
  4200. int x86_emulate_instruction(struct kvm_vcpu *vcpu,
  4201. unsigned long cr2,
  4202. int emulation_type,
  4203. void *insn,
  4204. int insn_len)
  4205. {
  4206. int r;
  4207. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  4208. bool writeback = true;
  4209. kvm_clear_exception_queue(vcpu);
  4210. if (!(emulation_type & EMULTYPE_NO_DECODE)) {
  4211. init_emulate_ctxt(vcpu);
  4212. ctxt->interruptibility = 0;
  4213. ctxt->have_exception = false;
  4214. ctxt->perm_ok = false;
  4215. ctxt->only_vendor_specific_insn
  4216. = emulation_type & EMULTYPE_TRAP_UD;
  4217. r = x86_decode_insn(ctxt, insn, insn_len);
  4218. trace_kvm_emulate_insn_start(vcpu);
  4219. ++vcpu->stat.insn_emulation;
  4220. if (r) {
  4221. if (emulation_type & EMULTYPE_TRAP_UD)
  4222. return EMULATE_FAIL;
  4223. if (reexecute_instruction(vcpu, cr2))
  4224. return EMULATE_DONE;
  4225. if (emulation_type & EMULTYPE_SKIP)
  4226. return EMULATE_FAIL;
  4227. return handle_emulation_failure(vcpu);
  4228. }
  4229. }
  4230. if (emulation_type & EMULTYPE_SKIP) {
  4231. kvm_rip_write(vcpu, ctxt->_eip);
  4232. return EMULATE_DONE;
  4233. }
  4234. /* this is needed for vmware backdoor interface to work since it
  4235. changes registers values during IO operation */
  4236. if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
  4237. vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
  4238. memcpy(ctxt->regs, vcpu->arch.regs, sizeof ctxt->regs);
  4239. }
  4240. restart:
  4241. r = x86_emulate_insn(ctxt);
  4242. if (r == EMULATION_INTERCEPTED)
  4243. return EMULATE_DONE;
  4244. if (r == EMULATION_FAILED) {
  4245. if (reexecute_instruction(vcpu, cr2))
  4246. return EMULATE_DONE;
  4247. return handle_emulation_failure(vcpu);
  4248. }
  4249. if (ctxt->have_exception) {
  4250. inject_emulated_exception(vcpu);
  4251. r = EMULATE_DONE;
  4252. } else if (vcpu->arch.pio.count) {
  4253. if (!vcpu->arch.pio.in)
  4254. vcpu->arch.pio.count = 0;
  4255. else
  4256. writeback = false;
  4257. r = EMULATE_DO_MMIO;
  4258. } else if (vcpu->mmio_needed) {
  4259. if (!vcpu->mmio_is_write)
  4260. writeback = false;
  4261. r = EMULATE_DO_MMIO;
  4262. } else if (r == EMULATION_RESTART)
  4263. goto restart;
  4264. else
  4265. r = EMULATE_DONE;
  4266. if (writeback) {
  4267. toggle_interruptibility(vcpu, ctxt->interruptibility);
  4268. kvm_set_rflags(vcpu, ctxt->eflags);
  4269. kvm_make_request(KVM_REQ_EVENT, vcpu);
  4270. memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
  4271. vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
  4272. kvm_rip_write(vcpu, ctxt->eip);
  4273. } else
  4274. vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
  4275. return r;
  4276. }
  4277. EXPORT_SYMBOL_GPL(x86_emulate_instruction);
  4278. int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
  4279. {
  4280. unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX);
  4281. int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
  4282. size, port, &val, 1);
  4283. /* do not return to emulator after return from userspace */
  4284. vcpu->arch.pio.count = 0;
  4285. return ret;
  4286. }
  4287. EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
  4288. static void tsc_bad(void *info)
  4289. {
  4290. __this_cpu_write(cpu_tsc_khz, 0);
  4291. }
  4292. static void tsc_khz_changed(void *data)
  4293. {
  4294. struct cpufreq_freqs *freq = data;
  4295. unsigned long khz = 0;
  4296. if (data)
  4297. khz = freq->new;
  4298. else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
  4299. khz = cpufreq_quick_get(raw_smp_processor_id());
  4300. if (!khz)
  4301. khz = tsc_khz;
  4302. __this_cpu_write(cpu_tsc_khz, khz);
  4303. }
  4304. static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
  4305. void *data)
  4306. {
  4307. struct cpufreq_freqs *freq = data;
  4308. struct kvm *kvm;
  4309. struct kvm_vcpu *vcpu;
  4310. int i, send_ipi = 0;
  4311. /*
  4312. * We allow guests to temporarily run on slowing clocks,
  4313. * provided we notify them after, or to run on accelerating
  4314. * clocks, provided we notify them before. Thus time never
  4315. * goes backwards.
  4316. *
  4317. * However, we have a problem. We can't atomically update
  4318. * the frequency of a given CPU from this function; it is
  4319. * merely a notifier, which can be called from any CPU.
  4320. * Changing the TSC frequency at arbitrary points in time
  4321. * requires a recomputation of local variables related to
  4322. * the TSC for each VCPU. We must flag these local variables
  4323. * to be updated and be sure the update takes place with the
  4324. * new frequency before any guests proceed.
  4325. *
  4326. * Unfortunately, the combination of hotplug CPU and frequency
  4327. * change creates an intractable locking scenario; the order
  4328. * of when these callouts happen is undefined with respect to
  4329. * CPU hotplug, and they can race with each other. As such,
  4330. * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
  4331. * undefined; you can actually have a CPU frequency change take
  4332. * place in between the computation of X and the setting of the
  4333. * variable. To protect against this problem, all updates of
  4334. * the per_cpu tsc_khz variable are done in an interrupt
  4335. * protected IPI, and all callers wishing to update the value
  4336. * must wait for a synchronous IPI to complete (which is trivial
  4337. * if the caller is on the CPU already). This establishes the
  4338. * necessary total order on variable updates.
  4339. *
  4340. * Note that because a guest time update may take place
  4341. * anytime after the setting of the VCPU's request bit, the
  4342. * correct TSC value must be set before the request. However,
  4343. * to ensure the update actually makes it to any guest which
  4344. * starts running in hardware virtualization between the set
  4345. * and the acquisition of the spinlock, we must also ping the
  4346. * CPU after setting the request bit.
  4347. *
  4348. */
  4349. if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
  4350. return 0;
  4351. if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
  4352. return 0;
  4353. smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
  4354. raw_spin_lock(&kvm_lock);
  4355. list_for_each_entry(kvm, &vm_list, vm_list) {
  4356. kvm_for_each_vcpu(i, vcpu, kvm) {
  4357. if (vcpu->cpu != freq->cpu)
  4358. continue;
  4359. kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
  4360. if (vcpu->cpu != smp_processor_id())
  4361. send_ipi = 1;
  4362. }
  4363. }
  4364. raw_spin_unlock(&kvm_lock);
  4365. if (freq->old < freq->new && send_ipi) {
  4366. /*
  4367. * We upscale the frequency. Must make the guest
  4368. * doesn't see old kvmclock values while running with
  4369. * the new frequency, otherwise we risk the guest sees
  4370. * time go backwards.
  4371. *
  4372. * In case we update the frequency for another cpu
  4373. * (which might be in guest context) send an interrupt
  4374. * to kick the cpu out of guest context. Next time
  4375. * guest context is entered kvmclock will be updated,
  4376. * so the guest will not see stale values.
  4377. */
  4378. smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
  4379. }
  4380. return 0;
  4381. }
  4382. static struct notifier_block kvmclock_cpufreq_notifier_block = {
  4383. .notifier_call = kvmclock_cpufreq_notifier
  4384. };
  4385. static int kvmclock_cpu_notifier(struct notifier_block *nfb,
  4386. unsigned long action, void *hcpu)
  4387. {
  4388. unsigned int cpu = (unsigned long)hcpu;
  4389. switch (action) {
  4390. case CPU_ONLINE:
  4391. case CPU_DOWN_FAILED:
  4392. smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
  4393. break;
  4394. case CPU_DOWN_PREPARE:
  4395. smp_call_function_single(cpu, tsc_bad, NULL, 1);
  4396. break;
  4397. }
  4398. return NOTIFY_OK;
  4399. }
  4400. static struct notifier_block kvmclock_cpu_notifier_block = {
  4401. .notifier_call = kvmclock_cpu_notifier,
  4402. .priority = -INT_MAX
  4403. };
  4404. static void kvm_timer_init(void)
  4405. {
  4406. int cpu;
  4407. max_tsc_khz = tsc_khz;
  4408. register_hotcpu_notifier(&kvmclock_cpu_notifier_block);
  4409. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
  4410. #ifdef CONFIG_CPU_FREQ
  4411. struct cpufreq_policy policy;
  4412. memset(&policy, 0, sizeof(policy));
  4413. cpu = get_cpu();
  4414. cpufreq_get_policy(&policy, cpu);
  4415. if (policy.cpuinfo.max_freq)
  4416. max_tsc_khz = policy.cpuinfo.max_freq;
  4417. put_cpu();
  4418. #endif
  4419. cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
  4420. CPUFREQ_TRANSITION_NOTIFIER);
  4421. }
  4422. pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz);
  4423. for_each_online_cpu(cpu)
  4424. smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
  4425. }
  4426. static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
  4427. static int kvm_is_in_guest(void)
  4428. {
  4429. return percpu_read(current_vcpu) != NULL;
  4430. }
  4431. static int kvm_is_user_mode(void)
  4432. {
  4433. int user_mode = 3;
  4434. if (percpu_read(current_vcpu))
  4435. user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
  4436. return user_mode != 0;
  4437. }
  4438. static unsigned long kvm_get_guest_ip(void)
  4439. {
  4440. unsigned long ip = 0;
  4441. if (percpu_read(current_vcpu))
  4442. ip = kvm_rip_read(percpu_read(current_vcpu));
  4443. return ip;
  4444. }
  4445. static struct perf_guest_info_callbacks kvm_guest_cbs = {
  4446. .is_in_guest = kvm_is_in_guest,
  4447. .is_user_mode = kvm_is_user_mode,
  4448. .get_guest_ip = kvm_get_guest_ip,
  4449. };
  4450. void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
  4451. {
  4452. percpu_write(current_vcpu, vcpu);
  4453. }
  4454. EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
  4455. void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
  4456. {
  4457. percpu_write(current_vcpu, NULL);
  4458. }
  4459. EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
  4460. static void kvm_set_mmio_spte_mask(void)
  4461. {
  4462. u64 mask;
  4463. int maxphyaddr = boot_cpu_data.x86_phys_bits;
  4464. /*
  4465. * Set the reserved bits and the present bit of an paging-structure
  4466. * entry to generate page fault with PFER.RSV = 1.
  4467. */
  4468. mask = ((1ull << (62 - maxphyaddr + 1)) - 1) << maxphyaddr;
  4469. mask |= 1ull;
  4470. #ifdef CONFIG_X86_64
  4471. /*
  4472. * If reserved bit is not supported, clear the present bit to disable
  4473. * mmio page fault.
  4474. */
  4475. if (maxphyaddr == 52)
  4476. mask &= ~1ull;
  4477. #endif
  4478. kvm_mmu_set_mmio_spte_mask(mask);
  4479. }
  4480. int kvm_arch_init(void *opaque)
  4481. {
  4482. int r;
  4483. struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;
  4484. if (kvm_x86_ops) {
  4485. printk(KERN_ERR "kvm: already loaded the other module\n");
  4486. r = -EEXIST;
  4487. goto out;
  4488. }
  4489. if (!ops->cpu_has_kvm_support()) {
  4490. printk(KERN_ERR "kvm: no hardware support\n");
  4491. r = -EOPNOTSUPP;
  4492. goto out;
  4493. }
  4494. if (ops->disabled_by_bios()) {
  4495. printk(KERN_ERR "kvm: disabled by bios\n");
  4496. r = -EOPNOTSUPP;
  4497. goto out;
  4498. }
  4499. r = kvm_mmu_module_init();
  4500. if (r)
  4501. goto out;
  4502. kvm_set_mmio_spte_mask();
  4503. kvm_init_msr_list();
  4504. kvm_x86_ops = ops;
  4505. kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
  4506. PT_DIRTY_MASK, PT64_NX_MASK, 0);
  4507. kvm_timer_init();
  4508. perf_register_guest_info_callbacks(&kvm_guest_cbs);
  4509. if (cpu_has_xsave)
  4510. host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
  4511. return 0;
  4512. out:
  4513. return r;
  4514. }
  4515. void kvm_arch_exit(void)
  4516. {
  4517. perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
  4518. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
  4519. cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
  4520. CPUFREQ_TRANSITION_NOTIFIER);
  4521. unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block);
  4522. kvm_x86_ops = NULL;
  4523. kvm_mmu_module_exit();
  4524. }
  4525. int kvm_emulate_halt(struct kvm_vcpu *vcpu)
  4526. {
  4527. ++vcpu->stat.halt_exits;
  4528. if (irqchip_in_kernel(vcpu->kvm)) {
  4529. vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
  4530. return 1;
  4531. } else {
  4532. vcpu->run->exit_reason = KVM_EXIT_HLT;
  4533. return 0;
  4534. }
  4535. }
  4536. EXPORT_SYMBOL_GPL(kvm_emulate_halt);
  4537. static inline gpa_t hc_gpa(struct kvm_vcpu *vcpu, unsigned long a0,
  4538. unsigned long a1)
  4539. {
  4540. if (is_long_mode(vcpu))
  4541. return a0;
  4542. else
  4543. return a0 | ((gpa_t)a1 << 32);
  4544. }
  4545. int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
  4546. {
  4547. u64 param, ingpa, outgpa, ret;
  4548. uint16_t code, rep_idx, rep_cnt, res = HV_STATUS_SUCCESS, rep_done = 0;
  4549. bool fast, longmode;
  4550. int cs_db, cs_l;
  4551. /*
  4552. * hypercall generates UD from non zero cpl and real mode
  4553. * per HYPER-V spec
  4554. */
  4555. if (kvm_x86_ops->get_cpl(vcpu) != 0 || !is_protmode(vcpu)) {
  4556. kvm_queue_exception(vcpu, UD_VECTOR);
  4557. return 0;
  4558. }
  4559. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  4560. longmode = is_long_mode(vcpu) && cs_l == 1;
  4561. if (!longmode) {
  4562. param = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDX) << 32) |
  4563. (kvm_register_read(vcpu, VCPU_REGS_RAX) & 0xffffffff);
  4564. ingpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RBX) << 32) |
  4565. (kvm_register_read(vcpu, VCPU_REGS_RCX) & 0xffffffff);
  4566. outgpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDI) << 32) |
  4567. (kvm_register_read(vcpu, VCPU_REGS_RSI) & 0xffffffff);
  4568. }
  4569. #ifdef CONFIG_X86_64
  4570. else {
  4571. param = kvm_register_read(vcpu, VCPU_REGS_RCX);
  4572. ingpa = kvm_register_read(vcpu, VCPU_REGS_RDX);
  4573. outgpa = kvm_register_read(vcpu, VCPU_REGS_R8);
  4574. }
  4575. #endif
  4576. code = param & 0xffff;
  4577. fast = (param >> 16) & 0x1;
  4578. rep_cnt = (param >> 32) & 0xfff;
  4579. rep_idx = (param >> 48) & 0xfff;
  4580. trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa);
  4581. switch (code) {
  4582. case HV_X64_HV_NOTIFY_LONG_SPIN_WAIT:
  4583. kvm_vcpu_on_spin(vcpu);
  4584. break;
  4585. default:
  4586. res = HV_STATUS_INVALID_HYPERCALL_CODE;
  4587. break;
  4588. }
  4589. ret = res | (((u64)rep_done & 0xfff) << 32);
  4590. if (longmode) {
  4591. kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
  4592. } else {
  4593. kvm_register_write(vcpu, VCPU_REGS_RDX, ret >> 32);
  4594. kvm_register_write(vcpu, VCPU_REGS_RAX, ret & 0xffffffff);
  4595. }
  4596. return 1;
  4597. }
  4598. int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
  4599. {
  4600. unsigned long nr, a0, a1, a2, a3, ret;
  4601. int r = 1;
  4602. if (kvm_hv_hypercall_enabled(vcpu->kvm))
  4603. return kvm_hv_hypercall(vcpu);
  4604. nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
  4605. a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
  4606. a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
  4607. a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
  4608. a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
  4609. trace_kvm_hypercall(nr, a0, a1, a2, a3);
  4610. if (!is_long_mode(vcpu)) {
  4611. nr &= 0xFFFFFFFF;
  4612. a0 &= 0xFFFFFFFF;
  4613. a1 &= 0xFFFFFFFF;
  4614. a2 &= 0xFFFFFFFF;
  4615. a3 &= 0xFFFFFFFF;
  4616. }
  4617. if (kvm_x86_ops->get_cpl(vcpu) != 0) {
  4618. ret = -KVM_EPERM;
  4619. goto out;
  4620. }
  4621. switch (nr) {
  4622. case KVM_HC_VAPIC_POLL_IRQ:
  4623. ret = 0;
  4624. break;
  4625. case KVM_HC_MMU_OP:
  4626. r = kvm_pv_mmu_op(vcpu, a0, hc_gpa(vcpu, a1, a2), &ret);
  4627. break;
  4628. default:
  4629. ret = -KVM_ENOSYS;
  4630. break;
  4631. }
  4632. out:
  4633. kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
  4634. ++vcpu->stat.hypercalls;
  4635. return r;
  4636. }
  4637. EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
  4638. int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
  4639. {
  4640. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  4641. char instruction[3];
  4642. unsigned long rip = kvm_rip_read(vcpu);
  4643. /*
  4644. * Blow out the MMU to ensure that no other VCPU has an active mapping
  4645. * to ensure that the updated hypercall appears atomically across all
  4646. * VCPUs.
  4647. */
  4648. kvm_mmu_zap_all(vcpu->kvm);
  4649. kvm_x86_ops->patch_hypercall(vcpu, instruction);
  4650. return emulator_write_emulated(ctxt, rip, instruction, 3, NULL);
  4651. }
  4652. static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
  4653. {
  4654. struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
  4655. int j, nent = vcpu->arch.cpuid_nent;
  4656. e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
  4657. /* when no next entry is found, the current entry[i] is reselected */
  4658. for (j = i + 1; ; j = (j + 1) % nent) {
  4659. struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j];
  4660. if (ej->function == e->function) {
  4661. ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
  4662. return j;
  4663. }
  4664. }
  4665. return 0; /* silence gcc, even though control never reaches here */
  4666. }
  4667. /* find an entry with matching function, matching index (if needed), and that
  4668. * should be read next (if it's stateful) */
  4669. static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
  4670. u32 function, u32 index)
  4671. {
  4672. if (e->function != function)
  4673. return 0;
  4674. if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
  4675. return 0;
  4676. if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
  4677. !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
  4678. return 0;
  4679. return 1;
  4680. }
  4681. struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
  4682. u32 function, u32 index)
  4683. {
  4684. int i;
  4685. struct kvm_cpuid_entry2 *best = NULL;
  4686. for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
  4687. struct kvm_cpuid_entry2 *e;
  4688. e = &vcpu->arch.cpuid_entries[i];
  4689. if (is_matching_cpuid_entry(e, function, index)) {
  4690. if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
  4691. move_to_next_stateful_cpuid_entry(vcpu, i);
  4692. best = e;
  4693. break;
  4694. }
  4695. }
  4696. return best;
  4697. }
  4698. EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
  4699. int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
  4700. {
  4701. struct kvm_cpuid_entry2 *best;
  4702. best = kvm_find_cpuid_entry(vcpu, 0x80000000, 0);
  4703. if (!best || best->eax < 0x80000008)
  4704. goto not_found;
  4705. best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
  4706. if (best)
  4707. return best->eax & 0xff;
  4708. not_found:
  4709. return 36;
  4710. }
  4711. /*
  4712. * If no match is found, check whether we exceed the vCPU's limit
  4713. * and return the content of the highest valid _standard_ leaf instead.
  4714. * This is to satisfy the CPUID specification.
  4715. */
  4716. static struct kvm_cpuid_entry2* check_cpuid_limit(struct kvm_vcpu *vcpu,
  4717. u32 function, u32 index)
  4718. {
  4719. struct kvm_cpuid_entry2 *maxlevel;
  4720. maxlevel = kvm_find_cpuid_entry(vcpu, function & 0x80000000, 0);
  4721. if (!maxlevel || maxlevel->eax >= function)
  4722. return NULL;
  4723. if (function & 0x80000000) {
  4724. maxlevel = kvm_find_cpuid_entry(vcpu, 0, 0);
  4725. if (!maxlevel)
  4726. return NULL;
  4727. }
  4728. return kvm_find_cpuid_entry(vcpu, maxlevel->eax, index);
  4729. }
  4730. void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
  4731. {
  4732. u32 function, index;
  4733. struct kvm_cpuid_entry2 *best;
  4734. function = kvm_register_read(vcpu, VCPU_REGS_RAX);
  4735. index = kvm_register_read(vcpu, VCPU_REGS_RCX);
  4736. kvm_register_write(vcpu, VCPU_REGS_RAX, 0);
  4737. kvm_register_write(vcpu, VCPU_REGS_RBX, 0);
  4738. kvm_register_write(vcpu, VCPU_REGS_RCX, 0);
  4739. kvm_register_write(vcpu, VCPU_REGS_RDX, 0);
  4740. best = kvm_find_cpuid_entry(vcpu, function, index);
  4741. if (!best)
  4742. best = check_cpuid_limit(vcpu, function, index);
  4743. if (best) {
  4744. kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax);
  4745. kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx);
  4746. kvm_register_write(vcpu, VCPU_REGS_RCX, best->ecx);
  4747. kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx);
  4748. }
  4749. kvm_x86_ops->skip_emulated_instruction(vcpu);
  4750. trace_kvm_cpuid(function,
  4751. kvm_register_read(vcpu, VCPU_REGS_RAX),
  4752. kvm_register_read(vcpu, VCPU_REGS_RBX),
  4753. kvm_register_read(vcpu, VCPU_REGS_RCX),
  4754. kvm_register_read(vcpu, VCPU_REGS_RDX));
  4755. }
  4756. EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
  4757. /*
  4758. * Check if userspace requested an interrupt window, and that the
  4759. * interrupt window is open.
  4760. *
  4761. * No need to exit to userspace if we already have an interrupt queued.
  4762. */
  4763. static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
  4764. {
  4765. return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
  4766. vcpu->run->request_interrupt_window &&
  4767. kvm_arch_interrupt_allowed(vcpu));
  4768. }
  4769. static void post_kvm_run_save(struct kvm_vcpu *vcpu)
  4770. {
  4771. struct kvm_run *kvm_run = vcpu->run;
  4772. kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
  4773. kvm_run->cr8 = kvm_get_cr8(vcpu);
  4774. kvm_run->apic_base = kvm_get_apic_base(vcpu);
  4775. if (irqchip_in_kernel(vcpu->kvm))
  4776. kvm_run->ready_for_interrupt_injection = 1;
  4777. else
  4778. kvm_run->ready_for_interrupt_injection =
  4779. kvm_arch_interrupt_allowed(vcpu) &&
  4780. !kvm_cpu_has_interrupt(vcpu) &&
  4781. !kvm_event_needs_reinjection(vcpu);
  4782. }
  4783. static void vapic_enter(struct kvm_vcpu *vcpu)
  4784. {
  4785. struct kvm_lapic *apic = vcpu->arch.apic;
  4786. struct page *page;
  4787. if (!apic || !apic->vapic_addr)
  4788. return;
  4789. page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
  4790. vcpu->arch.apic->vapic_page = page;
  4791. }
  4792. static void vapic_exit(struct kvm_vcpu *vcpu)
  4793. {
  4794. struct kvm_lapic *apic = vcpu->arch.apic;
  4795. int idx;
  4796. if (!apic || !apic->vapic_addr)
  4797. return;
  4798. idx = srcu_read_lock(&vcpu->kvm->srcu);
  4799. kvm_release_page_dirty(apic->vapic_page);
  4800. mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
  4801. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  4802. }
  4803. static void update_cr8_intercept(struct kvm_vcpu *vcpu)
  4804. {
  4805. int max_irr, tpr;
  4806. if (!kvm_x86_ops->update_cr8_intercept)
  4807. return;
  4808. if (!vcpu->arch.apic)
  4809. return;
  4810. if (!vcpu->arch.apic->vapic_addr)
  4811. max_irr = kvm_lapic_find_highest_irr(vcpu);
  4812. else
  4813. max_irr = -1;
  4814. if (max_irr != -1)
  4815. max_irr >>= 4;
  4816. tpr = kvm_lapic_get_cr8(vcpu);
  4817. kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
  4818. }
  4819. static void inject_pending_event(struct kvm_vcpu *vcpu)
  4820. {
  4821. /* try to reinject previous events if any */
  4822. if (vcpu->arch.exception.pending) {
  4823. trace_kvm_inj_exception(vcpu->arch.exception.nr,
  4824. vcpu->arch.exception.has_error_code,
  4825. vcpu->arch.exception.error_code);
  4826. kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
  4827. vcpu->arch.exception.has_error_code,
  4828. vcpu->arch.exception.error_code,
  4829. vcpu->arch.exception.reinject);
  4830. return;
  4831. }
  4832. if (vcpu->arch.nmi_injected) {
  4833. kvm_x86_ops->set_nmi(vcpu);
  4834. return;
  4835. }
  4836. if (vcpu->arch.interrupt.pending) {
  4837. kvm_x86_ops->set_irq(vcpu);
  4838. return;
  4839. }
  4840. /* try to inject new event if pending */
  4841. if (vcpu->arch.nmi_pending) {
  4842. if (kvm_x86_ops->nmi_allowed(vcpu)) {
  4843. vcpu->arch.nmi_pending = false;
  4844. vcpu->arch.nmi_injected = true;
  4845. kvm_x86_ops->set_nmi(vcpu);
  4846. }
  4847. } else if (kvm_cpu_has_interrupt(vcpu)) {
  4848. if (kvm_x86_ops->interrupt_allowed(vcpu)) {
  4849. kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
  4850. false);
  4851. kvm_x86_ops->set_irq(vcpu);
  4852. }
  4853. }
  4854. }
  4855. static void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
  4856. {
  4857. if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
  4858. !vcpu->guest_xcr0_loaded) {
  4859. /* kvm_set_xcr() also depends on this */
  4860. xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
  4861. vcpu->guest_xcr0_loaded = 1;
  4862. }
  4863. }
  4864. static void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
  4865. {
  4866. if (vcpu->guest_xcr0_loaded) {
  4867. if (vcpu->arch.xcr0 != host_xcr0)
  4868. xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
  4869. vcpu->guest_xcr0_loaded = 0;
  4870. }
  4871. }
  4872. static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
  4873. {
  4874. int r;
  4875. bool nmi_pending;
  4876. bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
  4877. vcpu->run->request_interrupt_window;
  4878. if (vcpu->requests) {
  4879. if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
  4880. kvm_mmu_unload(vcpu);
  4881. if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
  4882. __kvm_migrate_timers(vcpu);
  4883. if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
  4884. r = kvm_guest_time_update(vcpu);
  4885. if (unlikely(r))
  4886. goto out;
  4887. }
  4888. if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
  4889. kvm_mmu_sync_roots(vcpu);
  4890. if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
  4891. kvm_x86_ops->tlb_flush(vcpu);
  4892. if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
  4893. vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
  4894. r = 0;
  4895. goto out;
  4896. }
  4897. if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
  4898. vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
  4899. r = 0;
  4900. goto out;
  4901. }
  4902. if (kvm_check_request(KVM_REQ_DEACTIVATE_FPU, vcpu)) {
  4903. vcpu->fpu_active = 0;
  4904. kvm_x86_ops->fpu_deactivate(vcpu);
  4905. }
  4906. if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
  4907. /* Page is swapped out. Do synthetic halt */
  4908. vcpu->arch.apf.halted = true;
  4909. r = 1;
  4910. goto out;
  4911. }
  4912. if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
  4913. record_steal_time(vcpu);
  4914. }
  4915. r = kvm_mmu_reload(vcpu);
  4916. if (unlikely(r))
  4917. goto out;
  4918. /*
  4919. * An NMI can be injected between local nmi_pending read and
  4920. * vcpu->arch.nmi_pending read inside inject_pending_event().
  4921. * But in that case, KVM_REQ_EVENT will be set, which makes
  4922. * the race described above benign.
  4923. */
  4924. nmi_pending = ACCESS_ONCE(vcpu->arch.nmi_pending);
  4925. if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
  4926. inject_pending_event(vcpu);
  4927. /* enable NMI/IRQ window open exits if needed */
  4928. if (nmi_pending)
  4929. kvm_x86_ops->enable_nmi_window(vcpu);
  4930. else if (kvm_cpu_has_interrupt(vcpu) || req_int_win)
  4931. kvm_x86_ops->enable_irq_window(vcpu);
  4932. if (kvm_lapic_enabled(vcpu)) {
  4933. update_cr8_intercept(vcpu);
  4934. kvm_lapic_sync_to_vapic(vcpu);
  4935. }
  4936. }
  4937. preempt_disable();
  4938. kvm_x86_ops->prepare_guest_switch(vcpu);
  4939. if (vcpu->fpu_active)
  4940. kvm_load_guest_fpu(vcpu);
  4941. kvm_load_guest_xcr0(vcpu);
  4942. vcpu->mode = IN_GUEST_MODE;
  4943. /* We should set ->mode before check ->requests,
  4944. * see the comment in make_all_cpus_request.
  4945. */
  4946. smp_mb();
  4947. local_irq_disable();
  4948. if (vcpu->mode == EXITING_GUEST_MODE || vcpu->requests
  4949. || need_resched() || signal_pending(current)) {
  4950. vcpu->mode = OUTSIDE_GUEST_MODE;
  4951. smp_wmb();
  4952. local_irq_enable();
  4953. preempt_enable();
  4954. kvm_x86_ops->cancel_injection(vcpu);
  4955. r = 1;
  4956. goto out;
  4957. }
  4958. srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
  4959. kvm_guest_enter();
  4960. if (unlikely(vcpu->arch.switch_db_regs)) {
  4961. set_debugreg(0, 7);
  4962. set_debugreg(vcpu->arch.eff_db[0], 0);
  4963. set_debugreg(vcpu->arch.eff_db[1], 1);
  4964. set_debugreg(vcpu->arch.eff_db[2], 2);
  4965. set_debugreg(vcpu->arch.eff_db[3], 3);
  4966. }
  4967. trace_kvm_entry(vcpu->vcpu_id);
  4968. kvm_x86_ops->run(vcpu);
  4969. /*
  4970. * If the guest has used debug registers, at least dr7
  4971. * will be disabled while returning to the host.
  4972. * If we don't have active breakpoints in the host, we don't
  4973. * care about the messed up debug address registers. But if
  4974. * we have some of them active, restore the old state.
  4975. */
  4976. if (hw_breakpoint_active())
  4977. hw_breakpoint_restore();
  4978. kvm_get_msr(vcpu, MSR_IA32_TSC, &vcpu->arch.last_guest_tsc);
  4979. vcpu->mode = OUTSIDE_GUEST_MODE;
  4980. smp_wmb();
  4981. local_irq_enable();
  4982. ++vcpu->stat.exits;
  4983. /*
  4984. * We must have an instruction between local_irq_enable() and
  4985. * kvm_guest_exit(), so the timer interrupt isn't delayed by
  4986. * the interrupt shadow. The stat.exits increment will do nicely.
  4987. * But we need to prevent reordering, hence this barrier():
  4988. */
  4989. barrier();
  4990. kvm_guest_exit();
  4991. preempt_enable();
  4992. vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  4993. /*
  4994. * Profile KVM exit RIPs:
  4995. */
  4996. if (unlikely(prof_on == KVM_PROFILING)) {
  4997. unsigned long rip = kvm_rip_read(vcpu);
  4998. profile_hit(KVM_PROFILING, (void *)rip);
  4999. }
  5000. kvm_lapic_sync_from_vapic(vcpu);
  5001. r = kvm_x86_ops->handle_exit(vcpu);
  5002. out:
  5003. return r;
  5004. }
  5005. static int __vcpu_run(struct kvm_vcpu *vcpu)
  5006. {
  5007. int r;
  5008. struct kvm *kvm = vcpu->kvm;
  5009. if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) {
  5010. pr_debug("vcpu %d received sipi with vector # %x\n",
  5011. vcpu->vcpu_id, vcpu->arch.sipi_vector);
  5012. kvm_lapic_reset(vcpu);
  5013. r = kvm_arch_vcpu_reset(vcpu);
  5014. if (r)
  5015. return r;
  5016. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  5017. }
  5018. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  5019. vapic_enter(vcpu);
  5020. r = 1;
  5021. while (r > 0) {
  5022. if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
  5023. !vcpu->arch.apf.halted)
  5024. r = vcpu_enter_guest(vcpu);
  5025. else {
  5026. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  5027. kvm_vcpu_block(vcpu);
  5028. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  5029. if (kvm_check_request(KVM_REQ_UNHALT, vcpu))
  5030. {
  5031. switch(vcpu->arch.mp_state) {
  5032. case KVM_MP_STATE_HALTED:
  5033. vcpu->arch.mp_state =
  5034. KVM_MP_STATE_RUNNABLE;
  5035. case KVM_MP_STATE_RUNNABLE:
  5036. vcpu->arch.apf.halted = false;
  5037. break;
  5038. case KVM_MP_STATE_SIPI_RECEIVED:
  5039. default:
  5040. r = -EINTR;
  5041. break;
  5042. }
  5043. }
  5044. }
  5045. if (r <= 0)
  5046. break;
  5047. clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
  5048. if (kvm_cpu_has_pending_timer(vcpu))
  5049. kvm_inject_pending_timer_irqs(vcpu);
  5050. if (dm_request_for_irq_injection(vcpu)) {
  5051. r = -EINTR;
  5052. vcpu->run->exit_reason = KVM_EXIT_INTR;
  5053. ++vcpu->stat.request_irq_exits;
  5054. }
  5055. kvm_check_async_pf_completion(vcpu);
  5056. if (signal_pending(current)) {
  5057. r = -EINTR;
  5058. vcpu->run->exit_reason = KVM_EXIT_INTR;
  5059. ++vcpu->stat.signal_exits;
  5060. }
  5061. if (need_resched()) {
  5062. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  5063. kvm_resched(vcpu);
  5064. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  5065. }
  5066. }
  5067. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  5068. vapic_exit(vcpu);
  5069. return r;
  5070. }
  5071. static int complete_mmio(struct kvm_vcpu *vcpu)
  5072. {
  5073. struct kvm_run *run = vcpu->run;
  5074. int r;
  5075. if (!(vcpu->arch.pio.count || vcpu->mmio_needed))
  5076. return 1;
  5077. if (vcpu->mmio_needed) {
  5078. vcpu->mmio_needed = 0;
  5079. if (!vcpu->mmio_is_write)
  5080. memcpy(vcpu->mmio_data + vcpu->mmio_index,
  5081. run->mmio.data, 8);
  5082. vcpu->mmio_index += 8;
  5083. if (vcpu->mmio_index < vcpu->mmio_size) {
  5084. run->exit_reason = KVM_EXIT_MMIO;
  5085. run->mmio.phys_addr = vcpu->mmio_phys_addr + vcpu->mmio_index;
  5086. memcpy(run->mmio.data, vcpu->mmio_data + vcpu->mmio_index, 8);
  5087. run->mmio.len = min(vcpu->mmio_size - vcpu->mmio_index, 8);
  5088. run->mmio.is_write = vcpu->mmio_is_write;
  5089. vcpu->mmio_needed = 1;
  5090. return 0;
  5091. }
  5092. if (vcpu->mmio_is_write)
  5093. return 1;
  5094. vcpu->mmio_read_completed = 1;
  5095. }
  5096. vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  5097. r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
  5098. srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
  5099. if (r != EMULATE_DONE)
  5100. return 0;
  5101. return 1;
  5102. }
  5103. int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  5104. {
  5105. int r;
  5106. sigset_t sigsaved;
  5107. if (!tsk_used_math(current) && init_fpu(current))
  5108. return -ENOMEM;
  5109. if (vcpu->sigset_active)
  5110. sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
  5111. if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
  5112. kvm_vcpu_block(vcpu);
  5113. clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
  5114. r = -EAGAIN;
  5115. goto out;
  5116. }
  5117. /* re-sync apic's tpr */
  5118. if (!irqchip_in_kernel(vcpu->kvm)) {
  5119. if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
  5120. r = -EINVAL;
  5121. goto out;
  5122. }
  5123. }
  5124. r = complete_mmio(vcpu);
  5125. if (r <= 0)
  5126. goto out;
  5127. if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL)
  5128. kvm_register_write(vcpu, VCPU_REGS_RAX,
  5129. kvm_run->hypercall.ret);
  5130. r = __vcpu_run(vcpu);
  5131. out:
  5132. post_kvm_run_save(vcpu);
  5133. if (vcpu->sigset_active)
  5134. sigprocmask(SIG_SETMASK, &sigsaved, NULL);
  5135. return r;
  5136. }
  5137. int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  5138. {
  5139. if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
  5140. /*
  5141. * We are here if userspace calls get_regs() in the middle of
  5142. * instruction emulation. Registers state needs to be copied
  5143. * back from emulation context to vcpu. Usrapace shouldn't do
  5144. * that usually, but some bad designed PV devices (vmware
  5145. * backdoor interface) need this to work
  5146. */
  5147. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  5148. memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
  5149. vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
  5150. }
  5151. regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  5152. regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  5153. regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  5154. regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  5155. regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
  5156. regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
  5157. regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  5158. regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  5159. #ifdef CONFIG_X86_64
  5160. regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
  5161. regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
  5162. regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
  5163. regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
  5164. regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
  5165. regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
  5166. regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
  5167. regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
  5168. #endif
  5169. regs->rip = kvm_rip_read(vcpu);
  5170. regs->rflags = kvm_get_rflags(vcpu);
  5171. return 0;
  5172. }
  5173. int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  5174. {
  5175. vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
  5176. vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
  5177. kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
  5178. kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
  5179. kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
  5180. kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
  5181. kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
  5182. kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
  5183. kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
  5184. kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
  5185. #ifdef CONFIG_X86_64
  5186. kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
  5187. kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
  5188. kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
  5189. kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
  5190. kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
  5191. kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
  5192. kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
  5193. kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
  5194. #endif
  5195. kvm_rip_write(vcpu, regs->rip);
  5196. kvm_set_rflags(vcpu, regs->rflags);
  5197. vcpu->arch.exception.pending = false;
  5198. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5199. return 0;
  5200. }
  5201. void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
  5202. {
  5203. struct kvm_segment cs;
  5204. kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
  5205. *db = cs.db;
  5206. *l = cs.l;
  5207. }
  5208. EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
  5209. int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  5210. struct kvm_sregs *sregs)
  5211. {
  5212. struct desc_ptr dt;
  5213. kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  5214. kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  5215. kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  5216. kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  5217. kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  5218. kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  5219. kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  5220. kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  5221. kvm_x86_ops->get_idt(vcpu, &dt);
  5222. sregs->idt.limit = dt.size;
  5223. sregs->idt.base = dt.address;
  5224. kvm_x86_ops->get_gdt(vcpu, &dt);
  5225. sregs->gdt.limit = dt.size;
  5226. sregs->gdt.base = dt.address;
  5227. sregs->cr0 = kvm_read_cr0(vcpu);
  5228. sregs->cr2 = vcpu->arch.cr2;
  5229. sregs->cr3 = kvm_read_cr3(vcpu);
  5230. sregs->cr4 = kvm_read_cr4(vcpu);
  5231. sregs->cr8 = kvm_get_cr8(vcpu);
  5232. sregs->efer = vcpu->arch.efer;
  5233. sregs->apic_base = kvm_get_apic_base(vcpu);
  5234. memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
  5235. if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
  5236. set_bit(vcpu->arch.interrupt.nr,
  5237. (unsigned long *)sregs->interrupt_bitmap);
  5238. return 0;
  5239. }
  5240. int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
  5241. struct kvm_mp_state *mp_state)
  5242. {
  5243. mp_state->mp_state = vcpu->arch.mp_state;
  5244. return 0;
  5245. }
  5246. int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
  5247. struct kvm_mp_state *mp_state)
  5248. {
  5249. vcpu->arch.mp_state = mp_state->mp_state;
  5250. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5251. return 0;
  5252. }
  5253. int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason,
  5254. bool has_error_code, u32 error_code)
  5255. {
  5256. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  5257. int ret;
  5258. init_emulate_ctxt(vcpu);
  5259. ret = emulator_task_switch(ctxt, tss_selector, reason,
  5260. has_error_code, error_code);
  5261. if (ret)
  5262. return EMULATE_FAIL;
  5263. memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
  5264. kvm_rip_write(vcpu, ctxt->eip);
  5265. kvm_set_rflags(vcpu, ctxt->eflags);
  5266. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5267. return EMULATE_DONE;
  5268. }
  5269. EXPORT_SYMBOL_GPL(kvm_task_switch);
  5270. int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  5271. struct kvm_sregs *sregs)
  5272. {
  5273. int mmu_reset_needed = 0;
  5274. int pending_vec, max_bits, idx;
  5275. struct desc_ptr dt;
  5276. dt.size = sregs->idt.limit;
  5277. dt.address = sregs->idt.base;
  5278. kvm_x86_ops->set_idt(vcpu, &dt);
  5279. dt.size = sregs->gdt.limit;
  5280. dt.address = sregs->gdt.base;
  5281. kvm_x86_ops->set_gdt(vcpu, &dt);
  5282. vcpu->arch.cr2 = sregs->cr2;
  5283. mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
  5284. vcpu->arch.cr3 = sregs->cr3;
  5285. __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
  5286. kvm_set_cr8(vcpu, sregs->cr8);
  5287. mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
  5288. kvm_x86_ops->set_efer(vcpu, sregs->efer);
  5289. kvm_set_apic_base(vcpu, sregs->apic_base);
  5290. mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
  5291. kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
  5292. vcpu->arch.cr0 = sregs->cr0;
  5293. mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
  5294. kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
  5295. if (sregs->cr4 & X86_CR4_OSXSAVE)
  5296. update_cpuid(vcpu);
  5297. idx = srcu_read_lock(&vcpu->kvm->srcu);
  5298. if (!is_long_mode(vcpu) && is_pae(vcpu)) {
  5299. load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
  5300. mmu_reset_needed = 1;
  5301. }
  5302. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  5303. if (mmu_reset_needed)
  5304. kvm_mmu_reset_context(vcpu);
  5305. max_bits = (sizeof sregs->interrupt_bitmap) << 3;
  5306. pending_vec = find_first_bit(
  5307. (const unsigned long *)sregs->interrupt_bitmap, max_bits);
  5308. if (pending_vec < max_bits) {
  5309. kvm_queue_interrupt(vcpu, pending_vec, false);
  5310. pr_debug("Set back pending irq %d\n", pending_vec);
  5311. }
  5312. kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  5313. kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  5314. kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  5315. kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  5316. kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  5317. kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  5318. kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  5319. kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  5320. update_cr8_intercept(vcpu);
  5321. /* Older userspace won't unhalt the vcpu on reset. */
  5322. if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
  5323. sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
  5324. !is_protmode(vcpu))
  5325. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  5326. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5327. return 0;
  5328. }
  5329. int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
  5330. struct kvm_guest_debug *dbg)
  5331. {
  5332. unsigned long rflags;
  5333. int i, r;
  5334. if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
  5335. r = -EBUSY;
  5336. if (vcpu->arch.exception.pending)
  5337. goto out;
  5338. if (dbg->control & KVM_GUESTDBG_INJECT_DB)
  5339. kvm_queue_exception(vcpu, DB_VECTOR);
  5340. else
  5341. kvm_queue_exception(vcpu, BP_VECTOR);
  5342. }
  5343. /*
  5344. * Read rflags as long as potentially injected trace flags are still
  5345. * filtered out.
  5346. */
  5347. rflags = kvm_get_rflags(vcpu);
  5348. vcpu->guest_debug = dbg->control;
  5349. if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
  5350. vcpu->guest_debug = 0;
  5351. if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
  5352. for (i = 0; i < KVM_NR_DB_REGS; ++i)
  5353. vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
  5354. vcpu->arch.switch_db_regs =
  5355. (dbg->arch.debugreg[7] & DR7_BP_EN_MASK);
  5356. } else {
  5357. for (i = 0; i < KVM_NR_DB_REGS; i++)
  5358. vcpu->arch.eff_db[i] = vcpu->arch.db[i];
  5359. vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK);
  5360. }
  5361. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
  5362. vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
  5363. get_segment_base(vcpu, VCPU_SREG_CS);
  5364. /*
  5365. * Trigger an rflags update that will inject or remove the trace
  5366. * flags.
  5367. */
  5368. kvm_set_rflags(vcpu, rflags);
  5369. kvm_x86_ops->set_guest_debug(vcpu, dbg);
  5370. r = 0;
  5371. out:
  5372. return r;
  5373. }
  5374. /*
  5375. * Translate a guest virtual address to a guest physical address.
  5376. */
  5377. int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  5378. struct kvm_translation *tr)
  5379. {
  5380. unsigned long vaddr = tr->linear_address;
  5381. gpa_t gpa;
  5382. int idx;
  5383. idx = srcu_read_lock(&vcpu->kvm->srcu);
  5384. gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
  5385. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  5386. tr->physical_address = gpa;
  5387. tr->valid = gpa != UNMAPPED_GVA;
  5388. tr->writeable = 1;
  5389. tr->usermode = 0;
  5390. return 0;
  5391. }
  5392. int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  5393. {
  5394. struct i387_fxsave_struct *fxsave =
  5395. &vcpu->arch.guest_fpu.state->fxsave;
  5396. memcpy(fpu->fpr, fxsave->st_space, 128);
  5397. fpu->fcw = fxsave->cwd;
  5398. fpu->fsw = fxsave->swd;
  5399. fpu->ftwx = fxsave->twd;
  5400. fpu->last_opcode = fxsave->fop;
  5401. fpu->last_ip = fxsave->rip;
  5402. fpu->last_dp = fxsave->rdp;
  5403. memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
  5404. return 0;
  5405. }
  5406. int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  5407. {
  5408. struct i387_fxsave_struct *fxsave =
  5409. &vcpu->arch.guest_fpu.state->fxsave;
  5410. memcpy(fxsave->st_space, fpu->fpr, 128);
  5411. fxsave->cwd = fpu->fcw;
  5412. fxsave->swd = fpu->fsw;
  5413. fxsave->twd = fpu->ftwx;
  5414. fxsave->fop = fpu->last_opcode;
  5415. fxsave->rip = fpu->last_ip;
  5416. fxsave->rdp = fpu->last_dp;
  5417. memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
  5418. return 0;
  5419. }
  5420. int fx_init(struct kvm_vcpu *vcpu)
  5421. {
  5422. int err;
  5423. err = fpu_alloc(&vcpu->arch.guest_fpu);
  5424. if (err)
  5425. return err;
  5426. fpu_finit(&vcpu->arch.guest_fpu);
  5427. /*
  5428. * Ensure guest xcr0 is valid for loading
  5429. */
  5430. vcpu->arch.xcr0 = XSTATE_FP;
  5431. vcpu->arch.cr0 |= X86_CR0_ET;
  5432. return 0;
  5433. }
  5434. EXPORT_SYMBOL_GPL(fx_init);
  5435. static void fx_free(struct kvm_vcpu *vcpu)
  5436. {
  5437. fpu_free(&vcpu->arch.guest_fpu);
  5438. }
  5439. void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
  5440. {
  5441. if (vcpu->guest_fpu_loaded)
  5442. return;
  5443. /*
  5444. * Restore all possible states in the guest,
  5445. * and assume host would use all available bits.
  5446. * Guest xcr0 would be loaded later.
  5447. */
  5448. kvm_put_guest_xcr0(vcpu);
  5449. vcpu->guest_fpu_loaded = 1;
  5450. unlazy_fpu(current);
  5451. fpu_restore_checking(&vcpu->arch.guest_fpu);
  5452. trace_kvm_fpu(1);
  5453. }
  5454. void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
  5455. {
  5456. kvm_put_guest_xcr0(vcpu);
  5457. if (!vcpu->guest_fpu_loaded)
  5458. return;
  5459. vcpu->guest_fpu_loaded = 0;
  5460. fpu_save_init(&vcpu->arch.guest_fpu);
  5461. ++vcpu->stat.fpu_reload;
  5462. kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu);
  5463. trace_kvm_fpu(0);
  5464. }
  5465. void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
  5466. {
  5467. kvmclock_reset(vcpu);
  5468. free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
  5469. fx_free(vcpu);
  5470. kvm_x86_ops->vcpu_free(vcpu);
  5471. }
  5472. struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
  5473. unsigned int id)
  5474. {
  5475. if (check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
  5476. printk_once(KERN_WARNING
  5477. "kvm: SMP vm created on host with unstable TSC; "
  5478. "guest TSC will not be reliable\n");
  5479. return kvm_x86_ops->vcpu_create(kvm, id);
  5480. }
  5481. int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
  5482. {
  5483. int r;
  5484. vcpu->arch.mtrr_state.have_fixed = 1;
  5485. vcpu_load(vcpu);
  5486. r = kvm_arch_vcpu_reset(vcpu);
  5487. if (r == 0)
  5488. r = kvm_mmu_setup(vcpu);
  5489. vcpu_put(vcpu);
  5490. return r;
  5491. }
  5492. void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
  5493. {
  5494. vcpu->arch.apf.msr_val = 0;
  5495. vcpu_load(vcpu);
  5496. kvm_mmu_unload(vcpu);
  5497. vcpu_put(vcpu);
  5498. fx_free(vcpu);
  5499. kvm_x86_ops->vcpu_free(vcpu);
  5500. }
  5501. int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
  5502. {
  5503. vcpu->arch.nmi_pending = false;
  5504. vcpu->arch.nmi_injected = false;
  5505. vcpu->arch.switch_db_regs = 0;
  5506. memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
  5507. vcpu->arch.dr6 = DR6_FIXED_1;
  5508. vcpu->arch.dr7 = DR7_FIXED_1;
  5509. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5510. vcpu->arch.apf.msr_val = 0;
  5511. vcpu->arch.st.msr_val = 0;
  5512. kvmclock_reset(vcpu);
  5513. kvm_clear_async_pf_completion_queue(vcpu);
  5514. kvm_async_pf_hash_reset(vcpu);
  5515. vcpu->arch.apf.halted = false;
  5516. return kvm_x86_ops->vcpu_reset(vcpu);
  5517. }
  5518. int kvm_arch_hardware_enable(void *garbage)
  5519. {
  5520. struct kvm *kvm;
  5521. struct kvm_vcpu *vcpu;
  5522. int i;
  5523. kvm_shared_msr_cpu_online();
  5524. list_for_each_entry(kvm, &vm_list, vm_list)
  5525. kvm_for_each_vcpu(i, vcpu, kvm)
  5526. if (vcpu->cpu == smp_processor_id())
  5527. kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
  5528. return kvm_x86_ops->hardware_enable(garbage);
  5529. }
  5530. void kvm_arch_hardware_disable(void *garbage)
  5531. {
  5532. kvm_x86_ops->hardware_disable(garbage);
  5533. drop_user_return_notifiers(garbage);
  5534. }
  5535. int kvm_arch_hardware_setup(void)
  5536. {
  5537. return kvm_x86_ops->hardware_setup();
  5538. }
  5539. void kvm_arch_hardware_unsetup(void)
  5540. {
  5541. kvm_x86_ops->hardware_unsetup();
  5542. }
  5543. void kvm_arch_check_processor_compat(void *rtn)
  5544. {
  5545. kvm_x86_ops->check_processor_compatibility(rtn);
  5546. }
  5547. int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
  5548. {
  5549. struct page *page;
  5550. struct kvm *kvm;
  5551. int r;
  5552. BUG_ON(vcpu->kvm == NULL);
  5553. kvm = vcpu->kvm;
  5554. vcpu->arch.emulate_ctxt.ops = &emulate_ops;
  5555. vcpu->arch.walk_mmu = &vcpu->arch.mmu;
  5556. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  5557. vcpu->arch.mmu.translate_gpa = translate_gpa;
  5558. vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
  5559. if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
  5560. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  5561. else
  5562. vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
  5563. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  5564. if (!page) {
  5565. r = -ENOMEM;
  5566. goto fail;
  5567. }
  5568. vcpu->arch.pio_data = page_address(page);
  5569. kvm_init_tsc_catchup(vcpu, max_tsc_khz);
  5570. r = kvm_mmu_create(vcpu);
  5571. if (r < 0)
  5572. goto fail_free_pio_data;
  5573. if (irqchip_in_kernel(kvm)) {
  5574. r = kvm_create_lapic(vcpu);
  5575. if (r < 0)
  5576. goto fail_mmu_destroy;
  5577. }
  5578. vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
  5579. GFP_KERNEL);
  5580. if (!vcpu->arch.mce_banks) {
  5581. r = -ENOMEM;
  5582. goto fail_free_lapic;
  5583. }
  5584. vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
  5585. if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL))
  5586. goto fail_free_mce_banks;
  5587. kvm_async_pf_hash_reset(vcpu);
  5588. return 0;
  5589. fail_free_mce_banks:
  5590. kfree(vcpu->arch.mce_banks);
  5591. fail_free_lapic:
  5592. kvm_free_lapic(vcpu);
  5593. fail_mmu_destroy:
  5594. kvm_mmu_destroy(vcpu);
  5595. fail_free_pio_data:
  5596. free_page((unsigned long)vcpu->arch.pio_data);
  5597. fail:
  5598. return r;
  5599. }
  5600. void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
  5601. {
  5602. int idx;
  5603. kfree(vcpu->arch.mce_banks);
  5604. kvm_free_lapic(vcpu);
  5605. idx = srcu_read_lock(&vcpu->kvm->srcu);
  5606. kvm_mmu_destroy(vcpu);
  5607. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  5608. free_page((unsigned long)vcpu->arch.pio_data);
  5609. }
  5610. int kvm_arch_init_vm(struct kvm *kvm)
  5611. {
  5612. INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
  5613. INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
  5614. /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
  5615. set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
  5616. raw_spin_lock_init(&kvm->arch.tsc_write_lock);
  5617. return 0;
  5618. }
  5619. static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
  5620. {
  5621. vcpu_load(vcpu);
  5622. kvm_mmu_unload(vcpu);
  5623. vcpu_put(vcpu);
  5624. }
  5625. static void kvm_free_vcpus(struct kvm *kvm)
  5626. {
  5627. unsigned int i;
  5628. struct kvm_vcpu *vcpu;
  5629. /*
  5630. * Unpin any mmu pages first.
  5631. */
  5632. kvm_for_each_vcpu(i, vcpu, kvm) {
  5633. kvm_clear_async_pf_completion_queue(vcpu);
  5634. kvm_unload_vcpu_mmu(vcpu);
  5635. }
  5636. kvm_for_each_vcpu(i, vcpu, kvm)
  5637. kvm_arch_vcpu_free(vcpu);
  5638. mutex_lock(&kvm->lock);
  5639. for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
  5640. kvm->vcpus[i] = NULL;
  5641. atomic_set(&kvm->online_vcpus, 0);
  5642. mutex_unlock(&kvm->lock);
  5643. }
  5644. void kvm_arch_sync_events(struct kvm *kvm)
  5645. {
  5646. kvm_free_all_assigned_devices(kvm);
  5647. kvm_free_pit(kvm);
  5648. }
  5649. void kvm_arch_destroy_vm(struct kvm *kvm)
  5650. {
  5651. kvm_iommu_unmap_guest(kvm);
  5652. kfree(kvm->arch.vpic);
  5653. kfree(kvm->arch.vioapic);
  5654. kvm_free_vcpus(kvm);
  5655. if (kvm->arch.apic_access_page)
  5656. put_page(kvm->arch.apic_access_page);
  5657. if (kvm->arch.ept_identity_pagetable)
  5658. put_page(kvm->arch.ept_identity_pagetable);
  5659. }
  5660. int kvm_arch_prepare_memory_region(struct kvm *kvm,
  5661. struct kvm_memory_slot *memslot,
  5662. struct kvm_memory_slot old,
  5663. struct kvm_userspace_memory_region *mem,
  5664. int user_alloc)
  5665. {
  5666. int npages = memslot->npages;
  5667. int map_flags = MAP_PRIVATE | MAP_ANONYMOUS;
  5668. /* Prevent internal slot pages from being moved by fork()/COW. */
  5669. if (memslot->id >= KVM_MEMORY_SLOTS)
  5670. map_flags = MAP_SHARED | MAP_ANONYMOUS;
  5671. /*To keep backward compatibility with older userspace,
  5672. *x86 needs to hanlde !user_alloc case.
  5673. */
  5674. if (!user_alloc) {
  5675. if (npages && !old.rmap) {
  5676. unsigned long userspace_addr;
  5677. down_write(&current->mm->mmap_sem);
  5678. userspace_addr = do_mmap(NULL, 0,
  5679. npages * PAGE_SIZE,
  5680. PROT_READ | PROT_WRITE,
  5681. map_flags,
  5682. 0);
  5683. up_write(&current->mm->mmap_sem);
  5684. if (IS_ERR((void *)userspace_addr))
  5685. return PTR_ERR((void *)userspace_addr);
  5686. memslot->userspace_addr = userspace_addr;
  5687. }
  5688. }
  5689. return 0;
  5690. }
  5691. void kvm_arch_commit_memory_region(struct kvm *kvm,
  5692. struct kvm_userspace_memory_region *mem,
  5693. struct kvm_memory_slot old,
  5694. int user_alloc)
  5695. {
  5696. int nr_mmu_pages = 0, npages = mem->memory_size >> PAGE_SHIFT;
  5697. if (!user_alloc && !old.user_alloc && old.rmap && !npages) {
  5698. int ret;
  5699. down_write(&current->mm->mmap_sem);
  5700. ret = do_munmap(current->mm, old.userspace_addr,
  5701. old.npages * PAGE_SIZE);
  5702. up_write(&current->mm->mmap_sem);
  5703. if (ret < 0)
  5704. printk(KERN_WARNING
  5705. "kvm_vm_ioctl_set_memory_region: "
  5706. "failed to munmap memory\n");
  5707. }
  5708. if (!kvm->arch.n_requested_mmu_pages)
  5709. nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
  5710. spin_lock(&kvm->mmu_lock);
  5711. if (nr_mmu_pages)
  5712. kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
  5713. kvm_mmu_slot_remove_write_access(kvm, mem->slot);
  5714. spin_unlock(&kvm->mmu_lock);
  5715. }
  5716. void kvm_arch_flush_shadow(struct kvm *kvm)
  5717. {
  5718. kvm_mmu_zap_all(kvm);
  5719. kvm_reload_remote_mmus(kvm);
  5720. }
  5721. int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
  5722. {
  5723. return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
  5724. !vcpu->arch.apf.halted)
  5725. || !list_empty_careful(&vcpu->async_pf.done)
  5726. || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
  5727. || vcpu->arch.nmi_pending ||
  5728. (kvm_arch_interrupt_allowed(vcpu) &&
  5729. kvm_cpu_has_interrupt(vcpu));
  5730. }
  5731. void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
  5732. {
  5733. int me;
  5734. int cpu = vcpu->cpu;
  5735. if (waitqueue_active(&vcpu->wq)) {
  5736. wake_up_interruptible(&vcpu->wq);
  5737. ++vcpu->stat.halt_wakeup;
  5738. }
  5739. me = get_cpu();
  5740. if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
  5741. if (kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE)
  5742. smp_send_reschedule(cpu);
  5743. put_cpu();
  5744. }
  5745. int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
  5746. {
  5747. return kvm_x86_ops->interrupt_allowed(vcpu);
  5748. }
  5749. bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
  5750. {
  5751. unsigned long current_rip = kvm_rip_read(vcpu) +
  5752. get_segment_base(vcpu, VCPU_SREG_CS);
  5753. return current_rip == linear_rip;
  5754. }
  5755. EXPORT_SYMBOL_GPL(kvm_is_linear_rip);
  5756. unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
  5757. {
  5758. unsigned long rflags;
  5759. rflags = kvm_x86_ops->get_rflags(vcpu);
  5760. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
  5761. rflags &= ~X86_EFLAGS_TF;
  5762. return rflags;
  5763. }
  5764. EXPORT_SYMBOL_GPL(kvm_get_rflags);
  5765. void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  5766. {
  5767. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
  5768. kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
  5769. rflags |= X86_EFLAGS_TF;
  5770. kvm_x86_ops->set_rflags(vcpu, rflags);
  5771. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5772. }
  5773. EXPORT_SYMBOL_GPL(kvm_set_rflags);
  5774. void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
  5775. {
  5776. int r;
  5777. if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) ||
  5778. is_error_page(work->page))
  5779. return;
  5780. r = kvm_mmu_reload(vcpu);
  5781. if (unlikely(r))
  5782. return;
  5783. if (!vcpu->arch.mmu.direct_map &&
  5784. work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu))
  5785. return;
  5786. vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true);
  5787. }
  5788. static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
  5789. {
  5790. return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
  5791. }
  5792. static inline u32 kvm_async_pf_next_probe(u32 key)
  5793. {
  5794. return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
  5795. }
  5796. static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
  5797. {
  5798. u32 key = kvm_async_pf_hash_fn(gfn);
  5799. while (vcpu->arch.apf.gfns[key] != ~0)
  5800. key = kvm_async_pf_next_probe(key);
  5801. vcpu->arch.apf.gfns[key] = gfn;
  5802. }
  5803. static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
  5804. {
  5805. int i;
  5806. u32 key = kvm_async_pf_hash_fn(gfn);
  5807. for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
  5808. (vcpu->arch.apf.gfns[key] != gfn &&
  5809. vcpu->arch.apf.gfns[key] != ~0); i++)
  5810. key = kvm_async_pf_next_probe(key);
  5811. return key;
  5812. }
  5813. bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
  5814. {
  5815. return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
  5816. }
  5817. static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
  5818. {
  5819. u32 i, j, k;
  5820. i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
  5821. while (true) {
  5822. vcpu->arch.apf.gfns[i] = ~0;
  5823. do {
  5824. j = kvm_async_pf_next_probe(j);
  5825. if (vcpu->arch.apf.gfns[j] == ~0)
  5826. return;
  5827. k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
  5828. /*
  5829. * k lies cyclically in ]i,j]
  5830. * | i.k.j |
  5831. * |....j i.k.| or |.k..j i...|
  5832. */
  5833. } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
  5834. vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
  5835. i = j;
  5836. }
  5837. }
  5838. static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
  5839. {
  5840. return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
  5841. sizeof(val));
  5842. }
  5843. void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
  5844. struct kvm_async_pf *work)
  5845. {
  5846. struct x86_exception fault;
  5847. trace_kvm_async_pf_not_present(work->arch.token, work->gva);
  5848. kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
  5849. if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
  5850. (vcpu->arch.apf.send_user_only &&
  5851. kvm_x86_ops->get_cpl(vcpu) == 0))
  5852. kvm_make_request(KVM_REQ_APF_HALT, vcpu);
  5853. else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
  5854. fault.vector = PF_VECTOR;
  5855. fault.error_code_valid = true;
  5856. fault.error_code = 0;
  5857. fault.nested_page_fault = false;
  5858. fault.address = work->arch.token;
  5859. kvm_inject_page_fault(vcpu, &fault);
  5860. }
  5861. }
  5862. void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
  5863. struct kvm_async_pf *work)
  5864. {
  5865. struct x86_exception fault;
  5866. trace_kvm_async_pf_ready(work->arch.token, work->gva);
  5867. if (is_error_page(work->page))
  5868. work->arch.token = ~0; /* broadcast wakeup */
  5869. else
  5870. kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
  5871. if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) &&
  5872. !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
  5873. fault.vector = PF_VECTOR;
  5874. fault.error_code_valid = true;
  5875. fault.error_code = 0;
  5876. fault.nested_page_fault = false;
  5877. fault.address = work->arch.token;
  5878. kvm_inject_page_fault(vcpu, &fault);
  5879. }
  5880. vcpu->arch.apf.halted = false;
  5881. }
  5882. bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
  5883. {
  5884. if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
  5885. return true;
  5886. else
  5887. return !kvm_event_needs_reinjection(vcpu) &&
  5888. kvm_x86_ops->interrupt_allowed(vcpu);
  5889. }
  5890. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
  5891. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
  5892. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
  5893. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
  5894. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
  5895. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
  5896. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
  5897. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
  5898. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
  5899. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
  5900. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
  5901. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);