x86.c 183 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * derived from drivers/kvm/kvm_main.c
  5. *
  6. * Copyright (C) 2006 Qumranet, Inc.
  7. * Copyright (C) 2008 Qumranet, Inc.
  8. * Copyright IBM Corporation, 2008
  9. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  10. *
  11. * Authors:
  12. * Avi Kivity <avi@qumranet.com>
  13. * Yaniv Kamay <yaniv@qumranet.com>
  14. * Amit Shah <amit.shah@qumranet.com>
  15. * Ben-Ami Yassour <benami@il.ibm.com>
  16. *
  17. * This work is licensed under the terms of the GNU GPL, version 2. See
  18. * the COPYING file in the top-level directory.
  19. *
  20. */
  21. #include <linux/kvm_host.h>
  22. #include "irq.h"
  23. #include "mmu.h"
  24. #include "i8254.h"
  25. #include "tss.h"
  26. #include "kvm_cache_regs.h"
  27. #include "x86.h"
  28. #include "cpuid.h"
  29. #include <linux/clocksource.h>
  30. #include <linux/interrupt.h>
  31. #include <linux/kvm.h>
  32. #include <linux/fs.h>
  33. #include <linux/vmalloc.h>
  34. #include <linux/module.h>
  35. #include <linux/mman.h>
  36. #include <linux/highmem.h>
  37. #include <linux/iommu.h>
  38. #include <linux/intel-iommu.h>
  39. #include <linux/cpufreq.h>
  40. #include <linux/user-return-notifier.h>
  41. #include <linux/srcu.h>
  42. #include <linux/slab.h>
  43. #include <linux/perf_event.h>
  44. #include <linux/uaccess.h>
  45. #include <linux/hash.h>
  46. #include <linux/pci.h>
  47. #include <linux/timekeeper_internal.h>
  48. #include <linux/pvclock_gtod.h>
  49. #include <trace/events/kvm.h>
  50. #define CREATE_TRACE_POINTS
  51. #include "trace.h"
  52. #include <asm/debugreg.h>
  53. #include <asm/msr.h>
  54. #include <asm/desc.h>
  55. #include <asm/mtrr.h>
  56. #include <asm/mce.h>
  57. #include <asm/i387.h>
  58. #include <asm/fpu-internal.h> /* Ugh! */
  59. #include <asm/xcr.h>
  60. #include <asm/pvclock.h>
  61. #include <asm/div64.h>
  62. #define MAX_IO_MSRS 256
  63. #define KVM_MAX_MCE_BANKS 32
  64. #define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P)
  65. #define emul_to_vcpu(ctxt) \
  66. container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)
  67. /* EFER defaults:
  68. * - enable syscall per default because its emulated by KVM
  69. * - enable LME and LMA per default on 64 bit KVM
  70. */
  71. #ifdef CONFIG_X86_64
  72. static
  73. u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
  74. #else
  75. static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
  76. #endif
  77. #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  78. #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  79. static void update_cr8_intercept(struct kvm_vcpu *vcpu);
  80. static void process_nmi(struct kvm_vcpu *vcpu);
  81. struct kvm_x86_ops *kvm_x86_ops;
  82. EXPORT_SYMBOL_GPL(kvm_x86_ops);
  83. static bool ignore_msrs = 0;
  84. module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
  85. bool kvm_has_tsc_control;
  86. EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
  87. u32 kvm_max_guest_tsc_khz;
  88. EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
  89. /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
  90. static u32 tsc_tolerance_ppm = 250;
  91. module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
  92. #define KVM_NR_SHARED_MSRS 16
  93. struct kvm_shared_msrs_global {
  94. int nr;
  95. u32 msrs[KVM_NR_SHARED_MSRS];
  96. };
  97. struct kvm_shared_msrs {
  98. struct user_return_notifier urn;
  99. bool registered;
  100. struct kvm_shared_msr_values {
  101. u64 host;
  102. u64 curr;
  103. } values[KVM_NR_SHARED_MSRS];
  104. };
  105. static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
  106. static struct kvm_shared_msrs __percpu *shared_msrs;
  107. struct kvm_stats_debugfs_item debugfs_entries[] = {
  108. { "pf_fixed", VCPU_STAT(pf_fixed) },
  109. { "pf_guest", VCPU_STAT(pf_guest) },
  110. { "tlb_flush", VCPU_STAT(tlb_flush) },
  111. { "invlpg", VCPU_STAT(invlpg) },
  112. { "exits", VCPU_STAT(exits) },
  113. { "io_exits", VCPU_STAT(io_exits) },
  114. { "mmio_exits", VCPU_STAT(mmio_exits) },
  115. { "signal_exits", VCPU_STAT(signal_exits) },
  116. { "irq_window", VCPU_STAT(irq_window_exits) },
  117. { "nmi_window", VCPU_STAT(nmi_window_exits) },
  118. { "halt_exits", VCPU_STAT(halt_exits) },
  119. { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  120. { "hypercalls", VCPU_STAT(hypercalls) },
  121. { "request_irq", VCPU_STAT(request_irq_exits) },
  122. { "irq_exits", VCPU_STAT(irq_exits) },
  123. { "host_state_reload", VCPU_STAT(host_state_reload) },
  124. { "efer_reload", VCPU_STAT(efer_reload) },
  125. { "fpu_reload", VCPU_STAT(fpu_reload) },
  126. { "insn_emulation", VCPU_STAT(insn_emulation) },
  127. { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
  128. { "irq_injections", VCPU_STAT(irq_injections) },
  129. { "nmi_injections", VCPU_STAT(nmi_injections) },
  130. { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
  131. { "mmu_pte_write", VM_STAT(mmu_pte_write) },
  132. { "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
  133. { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
  134. { "mmu_flooded", VM_STAT(mmu_flooded) },
  135. { "mmu_recycled", VM_STAT(mmu_recycled) },
  136. { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
  137. { "mmu_unsync", VM_STAT(mmu_unsync) },
  138. { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
  139. { "largepages", VM_STAT(lpages) },
  140. { NULL }
  141. };
  142. u64 __read_mostly host_xcr0;
  143. static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
  144. static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
  145. {
  146. int i;
  147. for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
  148. vcpu->arch.apf.gfns[i] = ~0;
  149. }
  150. static void kvm_on_user_return(struct user_return_notifier *urn)
  151. {
  152. unsigned slot;
  153. struct kvm_shared_msrs *locals
  154. = container_of(urn, struct kvm_shared_msrs, urn);
  155. struct kvm_shared_msr_values *values;
  156. for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
  157. values = &locals->values[slot];
  158. if (values->host != values->curr) {
  159. wrmsrl(shared_msrs_global.msrs[slot], values->host);
  160. values->curr = values->host;
  161. }
  162. }
  163. locals->registered = false;
  164. user_return_notifier_unregister(urn);
  165. }
  166. static void shared_msr_update(unsigned slot, u32 msr)
  167. {
  168. u64 value;
  169. unsigned int cpu = smp_processor_id();
  170. struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
  171. /* only read, and nobody should modify it at this time,
  172. * so don't need lock */
  173. if (slot >= shared_msrs_global.nr) {
  174. printk(KERN_ERR "kvm: invalid MSR slot!");
  175. return;
  176. }
  177. rdmsrl_safe(msr, &value);
  178. smsr->values[slot].host = value;
  179. smsr->values[slot].curr = value;
  180. }
  181. void kvm_define_shared_msr(unsigned slot, u32 msr)
  182. {
  183. if (slot >= shared_msrs_global.nr)
  184. shared_msrs_global.nr = slot + 1;
  185. shared_msrs_global.msrs[slot] = msr;
  186. /* we need ensured the shared_msr_global have been updated */
  187. smp_wmb();
  188. }
  189. EXPORT_SYMBOL_GPL(kvm_define_shared_msr);
  190. static void kvm_shared_msr_cpu_online(void)
  191. {
  192. unsigned i;
  193. for (i = 0; i < shared_msrs_global.nr; ++i)
  194. shared_msr_update(i, shared_msrs_global.msrs[i]);
  195. }
  196. void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
  197. {
  198. unsigned int cpu = smp_processor_id();
  199. struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
  200. if (((value ^ smsr->values[slot].curr) & mask) == 0)
  201. return;
  202. smsr->values[slot].curr = value;
  203. wrmsrl(shared_msrs_global.msrs[slot], value);
  204. if (!smsr->registered) {
  205. smsr->urn.on_user_return = kvm_on_user_return;
  206. user_return_notifier_register(&smsr->urn);
  207. smsr->registered = true;
  208. }
  209. }
  210. EXPORT_SYMBOL_GPL(kvm_set_shared_msr);
  211. static void drop_user_return_notifiers(void *ignore)
  212. {
  213. unsigned int cpu = smp_processor_id();
  214. struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
  215. if (smsr->registered)
  216. kvm_on_user_return(&smsr->urn);
  217. }
  218. u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
  219. {
  220. return vcpu->arch.apic_base;
  221. }
  222. EXPORT_SYMBOL_GPL(kvm_get_apic_base);
  223. void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
  224. {
  225. /* TODO: reserve bits check */
  226. kvm_lapic_set_base(vcpu, data);
  227. }
  228. EXPORT_SYMBOL_GPL(kvm_set_apic_base);
  229. asmlinkage void kvm_spurious_fault(void)
  230. {
  231. /* Fault while not rebooting. We want the trace. */
  232. BUG();
  233. }
  234. EXPORT_SYMBOL_GPL(kvm_spurious_fault);
  235. #define EXCPT_BENIGN 0
  236. #define EXCPT_CONTRIBUTORY 1
  237. #define EXCPT_PF 2
  238. static int exception_class(int vector)
  239. {
  240. switch (vector) {
  241. case PF_VECTOR:
  242. return EXCPT_PF;
  243. case DE_VECTOR:
  244. case TS_VECTOR:
  245. case NP_VECTOR:
  246. case SS_VECTOR:
  247. case GP_VECTOR:
  248. return EXCPT_CONTRIBUTORY;
  249. default:
  250. break;
  251. }
  252. return EXCPT_BENIGN;
  253. }
  254. static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
  255. unsigned nr, bool has_error, u32 error_code,
  256. bool reinject)
  257. {
  258. u32 prev_nr;
  259. int class1, class2;
  260. kvm_make_request(KVM_REQ_EVENT, vcpu);
  261. if (!vcpu->arch.exception.pending) {
  262. queue:
  263. vcpu->arch.exception.pending = true;
  264. vcpu->arch.exception.has_error_code = has_error;
  265. vcpu->arch.exception.nr = nr;
  266. vcpu->arch.exception.error_code = error_code;
  267. vcpu->arch.exception.reinject = reinject;
  268. return;
  269. }
  270. /* to check exception */
  271. prev_nr = vcpu->arch.exception.nr;
  272. if (prev_nr == DF_VECTOR) {
  273. /* triple fault -> shutdown */
  274. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  275. return;
  276. }
  277. class1 = exception_class(prev_nr);
  278. class2 = exception_class(nr);
  279. if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
  280. || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
  281. /* generate double fault per SDM Table 5-5 */
  282. vcpu->arch.exception.pending = true;
  283. vcpu->arch.exception.has_error_code = true;
  284. vcpu->arch.exception.nr = DF_VECTOR;
  285. vcpu->arch.exception.error_code = 0;
  286. } else
  287. /* replace previous exception with a new one in a hope
  288. that instruction re-execution will regenerate lost
  289. exception */
  290. goto queue;
  291. }
  292. void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
  293. {
  294. kvm_multiple_exception(vcpu, nr, false, 0, false);
  295. }
  296. EXPORT_SYMBOL_GPL(kvm_queue_exception);
  297. void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
  298. {
  299. kvm_multiple_exception(vcpu, nr, false, 0, true);
  300. }
  301. EXPORT_SYMBOL_GPL(kvm_requeue_exception);
  302. void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
  303. {
  304. if (err)
  305. kvm_inject_gp(vcpu, 0);
  306. else
  307. kvm_x86_ops->skip_emulated_instruction(vcpu);
  308. }
  309. EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
  310. void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
  311. {
  312. ++vcpu->stat.pf_guest;
  313. vcpu->arch.cr2 = fault->address;
  314. kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
  315. }
  316. EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
  317. void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
  318. {
  319. if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
  320. vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
  321. else
  322. vcpu->arch.mmu.inject_page_fault(vcpu, fault);
  323. }
  324. void kvm_inject_nmi(struct kvm_vcpu *vcpu)
  325. {
  326. atomic_inc(&vcpu->arch.nmi_queued);
  327. kvm_make_request(KVM_REQ_NMI, vcpu);
  328. }
  329. EXPORT_SYMBOL_GPL(kvm_inject_nmi);
  330. void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
  331. {
  332. kvm_multiple_exception(vcpu, nr, true, error_code, false);
  333. }
  334. EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
  335. void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
  336. {
  337. kvm_multiple_exception(vcpu, nr, true, error_code, true);
  338. }
  339. EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);
  340. /*
  341. * Checks if cpl <= required_cpl; if true, return true. Otherwise queue
  342. * a #GP and return false.
  343. */
  344. bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
  345. {
  346. if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
  347. return true;
  348. kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
  349. return false;
  350. }
  351. EXPORT_SYMBOL_GPL(kvm_require_cpl);
  352. /*
  353. * This function will be used to read from the physical memory of the currently
  354. * running guest. The difference to kvm_read_guest_page is that this function
  355. * can read from guest physical or from the guest's guest physical memory.
  356. */
  357. int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
  358. gfn_t ngfn, void *data, int offset, int len,
  359. u32 access)
  360. {
  361. gfn_t real_gfn;
  362. gpa_t ngpa;
  363. ngpa = gfn_to_gpa(ngfn);
  364. real_gfn = mmu->translate_gpa(vcpu, ngpa, access);
  365. if (real_gfn == UNMAPPED_GVA)
  366. return -EFAULT;
  367. real_gfn = gpa_to_gfn(real_gfn);
  368. return kvm_read_guest_page(vcpu->kvm, real_gfn, data, offset, len);
  369. }
  370. EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);
  371. int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
  372. void *data, int offset, int len, u32 access)
  373. {
  374. return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
  375. data, offset, len, access);
  376. }
  377. /*
  378. * Load the pae pdptrs. Return true is they are all valid.
  379. */
  380. int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
  381. {
  382. gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
  383. unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
  384. int i;
  385. int ret;
  386. u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
  387. ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
  388. offset * sizeof(u64), sizeof(pdpte),
  389. PFERR_USER_MASK|PFERR_WRITE_MASK);
  390. if (ret < 0) {
  391. ret = 0;
  392. goto out;
  393. }
  394. for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
  395. if (is_present_gpte(pdpte[i]) &&
  396. (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
  397. ret = 0;
  398. goto out;
  399. }
  400. }
  401. ret = 1;
  402. memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
  403. __set_bit(VCPU_EXREG_PDPTR,
  404. (unsigned long *)&vcpu->arch.regs_avail);
  405. __set_bit(VCPU_EXREG_PDPTR,
  406. (unsigned long *)&vcpu->arch.regs_dirty);
  407. out:
  408. return ret;
  409. }
  410. EXPORT_SYMBOL_GPL(load_pdptrs);
  411. static bool pdptrs_changed(struct kvm_vcpu *vcpu)
  412. {
  413. u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
  414. bool changed = true;
  415. int offset;
  416. gfn_t gfn;
  417. int r;
  418. if (is_long_mode(vcpu) || !is_pae(vcpu))
  419. return false;
  420. if (!test_bit(VCPU_EXREG_PDPTR,
  421. (unsigned long *)&vcpu->arch.regs_avail))
  422. return true;
  423. gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT;
  424. offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1);
  425. r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
  426. PFERR_USER_MASK | PFERR_WRITE_MASK);
  427. if (r < 0)
  428. goto out;
  429. changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
  430. out:
  431. return changed;
  432. }
  433. int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  434. {
  435. unsigned long old_cr0 = kvm_read_cr0(vcpu);
  436. unsigned long update_bits = X86_CR0_PG | X86_CR0_WP |
  437. X86_CR0_CD | X86_CR0_NW;
  438. cr0 |= X86_CR0_ET;
  439. #ifdef CONFIG_X86_64
  440. if (cr0 & 0xffffffff00000000UL)
  441. return 1;
  442. #endif
  443. cr0 &= ~CR0_RESERVED_BITS;
  444. if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
  445. return 1;
  446. if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
  447. return 1;
  448. if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
  449. #ifdef CONFIG_X86_64
  450. if ((vcpu->arch.efer & EFER_LME)) {
  451. int cs_db, cs_l;
  452. if (!is_pae(vcpu))
  453. return 1;
  454. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  455. if (cs_l)
  456. return 1;
  457. } else
  458. #endif
  459. if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
  460. kvm_read_cr3(vcpu)))
  461. return 1;
  462. }
  463. if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
  464. return 1;
  465. kvm_x86_ops->set_cr0(vcpu, cr0);
  466. if ((cr0 ^ old_cr0) & X86_CR0_PG) {
  467. kvm_clear_async_pf_completion_queue(vcpu);
  468. kvm_async_pf_hash_reset(vcpu);
  469. }
  470. if ((cr0 ^ old_cr0) & update_bits)
  471. kvm_mmu_reset_context(vcpu);
  472. return 0;
  473. }
  474. EXPORT_SYMBOL_GPL(kvm_set_cr0);
  475. void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
  476. {
  477. (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
  478. }
  479. EXPORT_SYMBOL_GPL(kvm_lmsw);
  480. static void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
  481. {
  482. if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
  483. !vcpu->guest_xcr0_loaded) {
  484. /* kvm_set_xcr() also depends on this */
  485. xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
  486. vcpu->guest_xcr0_loaded = 1;
  487. }
  488. }
  489. static void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
  490. {
  491. if (vcpu->guest_xcr0_loaded) {
  492. if (vcpu->arch.xcr0 != host_xcr0)
  493. xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
  494. vcpu->guest_xcr0_loaded = 0;
  495. }
  496. }
  497. int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
  498. {
  499. u64 xcr0;
  500. /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now */
  501. if (index != XCR_XFEATURE_ENABLED_MASK)
  502. return 1;
  503. xcr0 = xcr;
  504. if (!(xcr0 & XSTATE_FP))
  505. return 1;
  506. if ((xcr0 & XSTATE_YMM) && !(xcr0 & XSTATE_SSE))
  507. return 1;
  508. if (xcr0 & ~host_xcr0)
  509. return 1;
  510. kvm_put_guest_xcr0(vcpu);
  511. vcpu->arch.xcr0 = xcr0;
  512. return 0;
  513. }
  514. int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
  515. {
  516. if (kvm_x86_ops->get_cpl(vcpu) != 0 ||
  517. __kvm_set_xcr(vcpu, index, xcr)) {
  518. kvm_inject_gp(vcpu, 0);
  519. return 1;
  520. }
  521. return 0;
  522. }
  523. EXPORT_SYMBOL_GPL(kvm_set_xcr);
  524. int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  525. {
  526. unsigned long old_cr4 = kvm_read_cr4(vcpu);
  527. unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE |
  528. X86_CR4_PAE | X86_CR4_SMEP;
  529. if (cr4 & CR4_RESERVED_BITS)
  530. return 1;
  531. if (!guest_cpuid_has_xsave(vcpu) && (cr4 & X86_CR4_OSXSAVE))
  532. return 1;
  533. if (!guest_cpuid_has_smep(vcpu) && (cr4 & X86_CR4_SMEP))
  534. return 1;
  535. if (!guest_cpuid_has_fsgsbase(vcpu) && (cr4 & X86_CR4_FSGSBASE))
  536. return 1;
  537. if (is_long_mode(vcpu)) {
  538. if (!(cr4 & X86_CR4_PAE))
  539. return 1;
  540. } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
  541. && ((cr4 ^ old_cr4) & pdptr_bits)
  542. && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
  543. kvm_read_cr3(vcpu)))
  544. return 1;
  545. if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
  546. if (!guest_cpuid_has_pcid(vcpu))
  547. return 1;
  548. /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
  549. if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
  550. return 1;
  551. }
  552. if (kvm_x86_ops->set_cr4(vcpu, cr4))
  553. return 1;
  554. if (((cr4 ^ old_cr4) & pdptr_bits) ||
  555. (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
  556. kvm_mmu_reset_context(vcpu);
  557. if ((cr4 ^ old_cr4) & X86_CR4_OSXSAVE)
  558. kvm_update_cpuid(vcpu);
  559. return 0;
  560. }
  561. EXPORT_SYMBOL_GPL(kvm_set_cr4);
  562. int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
  563. {
  564. if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
  565. kvm_mmu_sync_roots(vcpu);
  566. kvm_mmu_flush_tlb(vcpu);
  567. return 0;
  568. }
  569. if (is_long_mode(vcpu)) {
  570. if (kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)) {
  571. if (cr3 & CR3_PCID_ENABLED_RESERVED_BITS)
  572. return 1;
  573. } else
  574. if (cr3 & CR3_L_MODE_RESERVED_BITS)
  575. return 1;
  576. } else {
  577. if (is_pae(vcpu)) {
  578. if (cr3 & CR3_PAE_RESERVED_BITS)
  579. return 1;
  580. if (is_paging(vcpu) &&
  581. !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
  582. return 1;
  583. }
  584. /*
  585. * We don't check reserved bits in nonpae mode, because
  586. * this isn't enforced, and VMware depends on this.
  587. */
  588. }
  589. /*
  590. * Does the new cr3 value map to physical memory? (Note, we
  591. * catch an invalid cr3 even in real-mode, because it would
  592. * cause trouble later on when we turn on paging anyway.)
  593. *
  594. * A real CPU would silently accept an invalid cr3 and would
  595. * attempt to use it - with largely undefined (and often hard
  596. * to debug) behavior on the guest side.
  597. */
  598. if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
  599. return 1;
  600. vcpu->arch.cr3 = cr3;
  601. __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
  602. vcpu->arch.mmu.new_cr3(vcpu);
  603. return 0;
  604. }
  605. EXPORT_SYMBOL_GPL(kvm_set_cr3);
  606. int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
  607. {
  608. if (cr8 & CR8_RESERVED_BITS)
  609. return 1;
  610. if (irqchip_in_kernel(vcpu->kvm))
  611. kvm_lapic_set_tpr(vcpu, cr8);
  612. else
  613. vcpu->arch.cr8 = cr8;
  614. return 0;
  615. }
  616. EXPORT_SYMBOL_GPL(kvm_set_cr8);
  617. unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
  618. {
  619. if (irqchip_in_kernel(vcpu->kvm))
  620. return kvm_lapic_get_cr8(vcpu);
  621. else
  622. return vcpu->arch.cr8;
  623. }
  624. EXPORT_SYMBOL_GPL(kvm_get_cr8);
  625. static void kvm_update_dr7(struct kvm_vcpu *vcpu)
  626. {
  627. unsigned long dr7;
  628. if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
  629. dr7 = vcpu->arch.guest_debug_dr7;
  630. else
  631. dr7 = vcpu->arch.dr7;
  632. kvm_x86_ops->set_dr7(vcpu, dr7);
  633. vcpu->arch.switch_db_regs = (dr7 & DR7_BP_EN_MASK);
  634. }
  635. static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
  636. {
  637. switch (dr) {
  638. case 0 ... 3:
  639. vcpu->arch.db[dr] = val;
  640. if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
  641. vcpu->arch.eff_db[dr] = val;
  642. break;
  643. case 4:
  644. if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
  645. return 1; /* #UD */
  646. /* fall through */
  647. case 6:
  648. if (val & 0xffffffff00000000ULL)
  649. return -1; /* #GP */
  650. vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
  651. break;
  652. case 5:
  653. if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
  654. return 1; /* #UD */
  655. /* fall through */
  656. default: /* 7 */
  657. if (val & 0xffffffff00000000ULL)
  658. return -1; /* #GP */
  659. vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
  660. kvm_update_dr7(vcpu);
  661. break;
  662. }
  663. return 0;
  664. }
  665. int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
  666. {
  667. int res;
  668. res = __kvm_set_dr(vcpu, dr, val);
  669. if (res > 0)
  670. kvm_queue_exception(vcpu, UD_VECTOR);
  671. else if (res < 0)
  672. kvm_inject_gp(vcpu, 0);
  673. return res;
  674. }
  675. EXPORT_SYMBOL_GPL(kvm_set_dr);
  676. static int _kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
  677. {
  678. switch (dr) {
  679. case 0 ... 3:
  680. *val = vcpu->arch.db[dr];
  681. break;
  682. case 4:
  683. if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
  684. return 1;
  685. /* fall through */
  686. case 6:
  687. *val = vcpu->arch.dr6;
  688. break;
  689. case 5:
  690. if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
  691. return 1;
  692. /* fall through */
  693. default: /* 7 */
  694. *val = vcpu->arch.dr7;
  695. break;
  696. }
  697. return 0;
  698. }
  699. int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
  700. {
  701. if (_kvm_get_dr(vcpu, dr, val)) {
  702. kvm_queue_exception(vcpu, UD_VECTOR);
  703. return 1;
  704. }
  705. return 0;
  706. }
  707. EXPORT_SYMBOL_GPL(kvm_get_dr);
  708. bool kvm_rdpmc(struct kvm_vcpu *vcpu)
  709. {
  710. u32 ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  711. u64 data;
  712. int err;
  713. err = kvm_pmu_read_pmc(vcpu, ecx, &data);
  714. if (err)
  715. return err;
  716. kvm_register_write(vcpu, VCPU_REGS_RAX, (u32)data);
  717. kvm_register_write(vcpu, VCPU_REGS_RDX, data >> 32);
  718. return err;
  719. }
  720. EXPORT_SYMBOL_GPL(kvm_rdpmc);
  721. /*
  722. * List of msr numbers which we expose to userspace through KVM_GET_MSRS
  723. * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
  724. *
  725. * This list is modified at module load time to reflect the
  726. * capabilities of the host cpu. This capabilities test skips MSRs that are
  727. * kvm-specific. Those are put in the beginning of the list.
  728. */
  729. #define KVM_SAVE_MSRS_BEGIN 10
  730. static u32 msrs_to_save[] = {
  731. MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
  732. MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
  733. HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
  734. HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
  735. MSR_KVM_PV_EOI_EN,
  736. MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  737. MSR_STAR,
  738. #ifdef CONFIG_X86_64
  739. MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
  740. #endif
  741. MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA
  742. };
  743. static unsigned num_msrs_to_save;
  744. static const u32 emulated_msrs[] = {
  745. MSR_IA32_TSC_ADJUST,
  746. MSR_IA32_TSCDEADLINE,
  747. MSR_IA32_MISC_ENABLE,
  748. MSR_IA32_MCG_STATUS,
  749. MSR_IA32_MCG_CTL,
  750. };
  751. bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
  752. {
  753. if (efer & efer_reserved_bits)
  754. return false;
  755. if (efer & EFER_FFXSR) {
  756. struct kvm_cpuid_entry2 *feat;
  757. feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
  758. if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT)))
  759. return false;
  760. }
  761. if (efer & EFER_SVME) {
  762. struct kvm_cpuid_entry2 *feat;
  763. feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
  764. if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM)))
  765. return false;
  766. }
  767. return true;
  768. }
  769. EXPORT_SYMBOL_GPL(kvm_valid_efer);
  770. static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
  771. {
  772. u64 old_efer = vcpu->arch.efer;
  773. if (!kvm_valid_efer(vcpu, efer))
  774. return 1;
  775. if (is_paging(vcpu)
  776. && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
  777. return 1;
  778. efer &= ~EFER_LMA;
  779. efer |= vcpu->arch.efer & EFER_LMA;
  780. kvm_x86_ops->set_efer(vcpu, efer);
  781. /* Update reserved bits */
  782. if ((efer ^ old_efer) & EFER_NX)
  783. kvm_mmu_reset_context(vcpu);
  784. return 0;
  785. }
  786. void kvm_enable_efer_bits(u64 mask)
  787. {
  788. efer_reserved_bits &= ~mask;
  789. }
  790. EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
  791. /*
  792. * Writes msr value into into the appropriate "register".
  793. * Returns 0 on success, non-0 otherwise.
  794. * Assumes vcpu_load() was already called.
  795. */
  796. int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
  797. {
  798. return kvm_x86_ops->set_msr(vcpu, msr);
  799. }
  800. /*
  801. * Adapt set_msr() to msr_io()'s calling convention
  802. */
  803. static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
  804. {
  805. struct msr_data msr;
  806. msr.data = *data;
  807. msr.index = index;
  808. msr.host_initiated = true;
  809. return kvm_set_msr(vcpu, &msr);
  810. }
  811. #ifdef CONFIG_X86_64
  812. struct pvclock_gtod_data {
  813. seqcount_t seq;
  814. struct { /* extract of a clocksource struct */
  815. int vclock_mode;
  816. cycle_t cycle_last;
  817. cycle_t mask;
  818. u32 mult;
  819. u32 shift;
  820. } clock;
  821. /* open coded 'struct timespec' */
  822. u64 monotonic_time_snsec;
  823. time_t monotonic_time_sec;
  824. };
  825. static struct pvclock_gtod_data pvclock_gtod_data;
  826. static void update_pvclock_gtod(struct timekeeper *tk)
  827. {
  828. struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
  829. write_seqcount_begin(&vdata->seq);
  830. /* copy pvclock gtod data */
  831. vdata->clock.vclock_mode = tk->clock->archdata.vclock_mode;
  832. vdata->clock.cycle_last = tk->clock->cycle_last;
  833. vdata->clock.mask = tk->clock->mask;
  834. vdata->clock.mult = tk->mult;
  835. vdata->clock.shift = tk->shift;
  836. vdata->monotonic_time_sec = tk->xtime_sec
  837. + tk->wall_to_monotonic.tv_sec;
  838. vdata->monotonic_time_snsec = tk->xtime_nsec
  839. + (tk->wall_to_monotonic.tv_nsec
  840. << tk->shift);
  841. while (vdata->monotonic_time_snsec >=
  842. (((u64)NSEC_PER_SEC) << tk->shift)) {
  843. vdata->monotonic_time_snsec -=
  844. ((u64)NSEC_PER_SEC) << tk->shift;
  845. vdata->monotonic_time_sec++;
  846. }
  847. write_seqcount_end(&vdata->seq);
  848. }
  849. #endif
  850. static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
  851. {
  852. int version;
  853. int r;
  854. struct pvclock_wall_clock wc;
  855. struct timespec boot;
  856. if (!wall_clock)
  857. return;
  858. r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
  859. if (r)
  860. return;
  861. if (version & 1)
  862. ++version; /* first time write, random junk */
  863. ++version;
  864. kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
  865. /*
  866. * The guest calculates current wall clock time by adding
  867. * system time (updated by kvm_guest_time_update below) to the
  868. * wall clock specified here. guest system time equals host
  869. * system time for us, thus we must fill in host boot time here.
  870. */
  871. getboottime(&boot);
  872. if (kvm->arch.kvmclock_offset) {
  873. struct timespec ts = ns_to_timespec(kvm->arch.kvmclock_offset);
  874. boot = timespec_sub(boot, ts);
  875. }
  876. wc.sec = boot.tv_sec;
  877. wc.nsec = boot.tv_nsec;
  878. wc.version = version;
  879. kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
  880. version++;
  881. kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
  882. }
  883. static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
  884. {
  885. uint32_t quotient, remainder;
  886. /* Don't try to replace with do_div(), this one calculates
  887. * "(dividend << 32) / divisor" */
  888. __asm__ ( "divl %4"
  889. : "=a" (quotient), "=d" (remainder)
  890. : "0" (0), "1" (dividend), "r" (divisor) );
  891. return quotient;
  892. }
  893. static void kvm_get_time_scale(uint32_t scaled_khz, uint32_t base_khz,
  894. s8 *pshift, u32 *pmultiplier)
  895. {
  896. uint64_t scaled64;
  897. int32_t shift = 0;
  898. uint64_t tps64;
  899. uint32_t tps32;
  900. tps64 = base_khz * 1000LL;
  901. scaled64 = scaled_khz * 1000LL;
  902. while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
  903. tps64 >>= 1;
  904. shift--;
  905. }
  906. tps32 = (uint32_t)tps64;
  907. while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
  908. if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
  909. scaled64 >>= 1;
  910. else
  911. tps32 <<= 1;
  912. shift++;
  913. }
  914. *pshift = shift;
  915. *pmultiplier = div_frac(scaled64, tps32);
  916. pr_debug("%s: base_khz %u => %u, shift %d, mul %u\n",
  917. __func__, base_khz, scaled_khz, shift, *pmultiplier);
  918. }
  919. static inline u64 get_kernel_ns(void)
  920. {
  921. struct timespec ts;
  922. WARN_ON(preemptible());
  923. ktime_get_ts(&ts);
  924. monotonic_to_bootbased(&ts);
  925. return timespec_to_ns(&ts);
  926. }
  927. #ifdef CONFIG_X86_64
  928. static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
  929. #endif
  930. static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
  931. unsigned long max_tsc_khz;
  932. static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
  933. {
  934. return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
  935. vcpu->arch.virtual_tsc_shift);
  936. }
  937. static u32 adjust_tsc_khz(u32 khz, s32 ppm)
  938. {
  939. u64 v = (u64)khz * (1000000 + ppm);
  940. do_div(v, 1000000);
  941. return v;
  942. }
  943. static void kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 this_tsc_khz)
  944. {
  945. u32 thresh_lo, thresh_hi;
  946. int use_scaling = 0;
  947. /* tsc_khz can be zero if TSC calibration fails */
  948. if (this_tsc_khz == 0)
  949. return;
  950. /* Compute a scale to convert nanoseconds in TSC cycles */
  951. kvm_get_time_scale(this_tsc_khz, NSEC_PER_SEC / 1000,
  952. &vcpu->arch.virtual_tsc_shift,
  953. &vcpu->arch.virtual_tsc_mult);
  954. vcpu->arch.virtual_tsc_khz = this_tsc_khz;
  955. /*
  956. * Compute the variation in TSC rate which is acceptable
  957. * within the range of tolerance and decide if the
  958. * rate being applied is within that bounds of the hardware
  959. * rate. If so, no scaling or compensation need be done.
  960. */
  961. thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
  962. thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
  963. if (this_tsc_khz < thresh_lo || this_tsc_khz > thresh_hi) {
  964. pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", this_tsc_khz, thresh_lo, thresh_hi);
  965. use_scaling = 1;
  966. }
  967. kvm_x86_ops->set_tsc_khz(vcpu, this_tsc_khz, use_scaling);
  968. }
  969. static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
  970. {
  971. u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
  972. vcpu->arch.virtual_tsc_mult,
  973. vcpu->arch.virtual_tsc_shift);
  974. tsc += vcpu->arch.this_tsc_write;
  975. return tsc;
  976. }
  977. void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
  978. {
  979. #ifdef CONFIG_X86_64
  980. bool vcpus_matched;
  981. bool do_request = false;
  982. struct kvm_arch *ka = &vcpu->kvm->arch;
  983. struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
  984. vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
  985. atomic_read(&vcpu->kvm->online_vcpus));
  986. if (vcpus_matched && gtod->clock.vclock_mode == VCLOCK_TSC)
  987. if (!ka->use_master_clock)
  988. do_request = 1;
  989. if (!vcpus_matched && ka->use_master_clock)
  990. do_request = 1;
  991. if (do_request)
  992. kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
  993. trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
  994. atomic_read(&vcpu->kvm->online_vcpus),
  995. ka->use_master_clock, gtod->clock.vclock_mode);
  996. #endif
  997. }
  998. static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
  999. {
  1000. u64 curr_offset = kvm_x86_ops->read_tsc_offset(vcpu);
  1001. vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
  1002. }
  1003. void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
  1004. {
  1005. struct kvm *kvm = vcpu->kvm;
  1006. u64 offset, ns, elapsed;
  1007. unsigned long flags;
  1008. s64 usdiff;
  1009. bool matched;
  1010. u64 data = msr->data;
  1011. raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
  1012. offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
  1013. ns = get_kernel_ns();
  1014. elapsed = ns - kvm->arch.last_tsc_nsec;
  1015. if (vcpu->arch.virtual_tsc_khz) {
  1016. int faulted = 0;
  1017. /* n.b - signed multiplication and division required */
  1018. usdiff = data - kvm->arch.last_tsc_write;
  1019. #ifdef CONFIG_X86_64
  1020. usdiff = (usdiff * 1000) / vcpu->arch.virtual_tsc_khz;
  1021. #else
  1022. /* do_div() only does unsigned */
  1023. asm("1: idivl %[divisor]\n"
  1024. "2: xor %%edx, %%edx\n"
  1025. " movl $0, %[faulted]\n"
  1026. "3:\n"
  1027. ".section .fixup,\"ax\"\n"
  1028. "4: movl $1, %[faulted]\n"
  1029. " jmp 3b\n"
  1030. ".previous\n"
  1031. _ASM_EXTABLE(1b, 4b)
  1032. : "=A"(usdiff), [faulted] "=r" (faulted)
  1033. : "A"(usdiff * 1000), [divisor] "rm"(vcpu->arch.virtual_tsc_khz));
  1034. #endif
  1035. do_div(elapsed, 1000);
  1036. usdiff -= elapsed;
  1037. if (usdiff < 0)
  1038. usdiff = -usdiff;
  1039. /* idivl overflow => difference is larger than USEC_PER_SEC */
  1040. if (faulted)
  1041. usdiff = USEC_PER_SEC;
  1042. } else
  1043. usdiff = USEC_PER_SEC; /* disable TSC match window below */
  1044. /*
  1045. * Special case: TSC write with a small delta (1 second) of virtual
  1046. * cycle time against real time is interpreted as an attempt to
  1047. * synchronize the CPU.
  1048. *
  1049. * For a reliable TSC, we can match TSC offsets, and for an unstable
  1050. * TSC, we add elapsed time in this computation. We could let the
  1051. * compensation code attempt to catch up if we fall behind, but
  1052. * it's better to try to match offsets from the beginning.
  1053. */
  1054. if (usdiff < USEC_PER_SEC &&
  1055. vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
  1056. if (!check_tsc_unstable()) {
  1057. offset = kvm->arch.cur_tsc_offset;
  1058. pr_debug("kvm: matched tsc offset for %llu\n", data);
  1059. } else {
  1060. u64 delta = nsec_to_cycles(vcpu, elapsed);
  1061. data += delta;
  1062. offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
  1063. pr_debug("kvm: adjusted tsc offset by %llu\n", delta);
  1064. }
  1065. matched = true;
  1066. } else {
  1067. /*
  1068. * We split periods of matched TSC writes into generations.
  1069. * For each generation, we track the original measured
  1070. * nanosecond time, offset, and write, so if TSCs are in
  1071. * sync, we can match exact offset, and if not, we can match
  1072. * exact software computation in compute_guest_tsc()
  1073. *
  1074. * These values are tracked in kvm->arch.cur_xxx variables.
  1075. */
  1076. kvm->arch.cur_tsc_generation++;
  1077. kvm->arch.cur_tsc_nsec = ns;
  1078. kvm->arch.cur_tsc_write = data;
  1079. kvm->arch.cur_tsc_offset = offset;
  1080. matched = false;
  1081. pr_debug("kvm: new tsc generation %u, clock %llu\n",
  1082. kvm->arch.cur_tsc_generation, data);
  1083. }
  1084. /*
  1085. * We also track th most recent recorded KHZ, write and time to
  1086. * allow the matching interval to be extended at each write.
  1087. */
  1088. kvm->arch.last_tsc_nsec = ns;
  1089. kvm->arch.last_tsc_write = data;
  1090. kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
  1091. /* Reset of TSC must disable overshoot protection below */
  1092. vcpu->arch.hv_clock.tsc_timestamp = 0;
  1093. vcpu->arch.last_guest_tsc = data;
  1094. /* Keep track of which generation this VCPU has synchronized to */
  1095. vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
  1096. vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
  1097. vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
  1098. if (guest_cpuid_has_tsc_adjust(vcpu) && !msr->host_initiated)
  1099. update_ia32_tsc_adjust_msr(vcpu, offset);
  1100. kvm_x86_ops->write_tsc_offset(vcpu, offset);
  1101. raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
  1102. spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
  1103. if (matched)
  1104. kvm->arch.nr_vcpus_matched_tsc++;
  1105. else
  1106. kvm->arch.nr_vcpus_matched_tsc = 0;
  1107. kvm_track_tsc_matching(vcpu);
  1108. spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
  1109. }
  1110. EXPORT_SYMBOL_GPL(kvm_write_tsc);
  1111. #ifdef CONFIG_X86_64
  1112. static cycle_t read_tsc(void)
  1113. {
  1114. cycle_t ret;
  1115. u64 last;
  1116. /*
  1117. * Empirically, a fence (of type that depends on the CPU)
  1118. * before rdtsc is enough to ensure that rdtsc is ordered
  1119. * with respect to loads. The various CPU manuals are unclear
  1120. * as to whether rdtsc can be reordered with later loads,
  1121. * but no one has ever seen it happen.
  1122. */
  1123. rdtsc_barrier();
  1124. ret = (cycle_t)vget_cycles();
  1125. last = pvclock_gtod_data.clock.cycle_last;
  1126. if (likely(ret >= last))
  1127. return ret;
  1128. /*
  1129. * GCC likes to generate cmov here, but this branch is extremely
  1130. * predictable (it's just a funciton of time and the likely is
  1131. * very likely) and there's a data dependence, so force GCC
  1132. * to generate a branch instead. I don't barrier() because
  1133. * we don't actually need a barrier, and if this function
  1134. * ever gets inlined it will generate worse code.
  1135. */
  1136. asm volatile ("");
  1137. return last;
  1138. }
  1139. static inline u64 vgettsc(cycle_t *cycle_now)
  1140. {
  1141. long v;
  1142. struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
  1143. *cycle_now = read_tsc();
  1144. v = (*cycle_now - gtod->clock.cycle_last) & gtod->clock.mask;
  1145. return v * gtod->clock.mult;
  1146. }
  1147. static int do_monotonic(struct timespec *ts, cycle_t *cycle_now)
  1148. {
  1149. unsigned long seq;
  1150. u64 ns;
  1151. int mode;
  1152. struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
  1153. ts->tv_nsec = 0;
  1154. do {
  1155. seq = read_seqcount_begin(&gtod->seq);
  1156. mode = gtod->clock.vclock_mode;
  1157. ts->tv_sec = gtod->monotonic_time_sec;
  1158. ns = gtod->monotonic_time_snsec;
  1159. ns += vgettsc(cycle_now);
  1160. ns >>= gtod->clock.shift;
  1161. } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
  1162. timespec_add_ns(ts, ns);
  1163. return mode;
  1164. }
  1165. /* returns true if host is using tsc clocksource */
  1166. static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now)
  1167. {
  1168. struct timespec ts;
  1169. /* checked again under seqlock below */
  1170. if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
  1171. return false;
  1172. if (do_monotonic(&ts, cycle_now) != VCLOCK_TSC)
  1173. return false;
  1174. monotonic_to_bootbased(&ts);
  1175. *kernel_ns = timespec_to_ns(&ts);
  1176. return true;
  1177. }
  1178. #endif
  1179. /*
  1180. *
  1181. * Assuming a stable TSC across physical CPUS, and a stable TSC
  1182. * across virtual CPUs, the following condition is possible.
  1183. * Each numbered line represents an event visible to both
  1184. * CPUs at the next numbered event.
  1185. *
  1186. * "timespecX" represents host monotonic time. "tscX" represents
  1187. * RDTSC value.
  1188. *
  1189. * VCPU0 on CPU0 | VCPU1 on CPU1
  1190. *
  1191. * 1. read timespec0,tsc0
  1192. * 2. | timespec1 = timespec0 + N
  1193. * | tsc1 = tsc0 + M
  1194. * 3. transition to guest | transition to guest
  1195. * 4. ret0 = timespec0 + (rdtsc - tsc0) |
  1196. * 5. | ret1 = timespec1 + (rdtsc - tsc1)
  1197. * | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
  1198. *
  1199. * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
  1200. *
  1201. * - ret0 < ret1
  1202. * - timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
  1203. * ...
  1204. * - 0 < N - M => M < N
  1205. *
  1206. * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
  1207. * always the case (the difference between two distinct xtime instances
  1208. * might be smaller then the difference between corresponding TSC reads,
  1209. * when updating guest vcpus pvclock areas).
  1210. *
  1211. * To avoid that problem, do not allow visibility of distinct
  1212. * system_timestamp/tsc_timestamp values simultaneously: use a master
  1213. * copy of host monotonic time values. Update that master copy
  1214. * in lockstep.
  1215. *
  1216. * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
  1217. *
  1218. */
  1219. static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
  1220. {
  1221. #ifdef CONFIG_X86_64
  1222. struct kvm_arch *ka = &kvm->arch;
  1223. int vclock_mode;
  1224. bool host_tsc_clocksource, vcpus_matched;
  1225. vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
  1226. atomic_read(&kvm->online_vcpus));
  1227. /*
  1228. * If the host uses TSC clock, then passthrough TSC as stable
  1229. * to the guest.
  1230. */
  1231. host_tsc_clocksource = kvm_get_time_and_clockread(
  1232. &ka->master_kernel_ns,
  1233. &ka->master_cycle_now);
  1234. ka->use_master_clock = host_tsc_clocksource & vcpus_matched;
  1235. if (ka->use_master_clock)
  1236. atomic_set(&kvm_guest_has_master_clock, 1);
  1237. vclock_mode = pvclock_gtod_data.clock.vclock_mode;
  1238. trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
  1239. vcpus_matched);
  1240. #endif
  1241. }
  1242. static int kvm_guest_time_update(struct kvm_vcpu *v)
  1243. {
  1244. unsigned long flags, this_tsc_khz;
  1245. struct kvm_vcpu_arch *vcpu = &v->arch;
  1246. struct kvm_arch *ka = &v->kvm->arch;
  1247. s64 kernel_ns, max_kernel_ns;
  1248. u64 tsc_timestamp, host_tsc;
  1249. struct pvclock_vcpu_time_info guest_hv_clock;
  1250. u8 pvclock_flags;
  1251. bool use_master_clock;
  1252. kernel_ns = 0;
  1253. host_tsc = 0;
  1254. /*
  1255. * If the host uses TSC clock, then passthrough TSC as stable
  1256. * to the guest.
  1257. */
  1258. spin_lock(&ka->pvclock_gtod_sync_lock);
  1259. use_master_clock = ka->use_master_clock;
  1260. if (use_master_clock) {
  1261. host_tsc = ka->master_cycle_now;
  1262. kernel_ns = ka->master_kernel_ns;
  1263. }
  1264. spin_unlock(&ka->pvclock_gtod_sync_lock);
  1265. /* Keep irq disabled to prevent changes to the clock */
  1266. local_irq_save(flags);
  1267. this_tsc_khz = __get_cpu_var(cpu_tsc_khz);
  1268. if (unlikely(this_tsc_khz == 0)) {
  1269. local_irq_restore(flags);
  1270. kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
  1271. return 1;
  1272. }
  1273. if (!use_master_clock) {
  1274. host_tsc = native_read_tsc();
  1275. kernel_ns = get_kernel_ns();
  1276. }
  1277. tsc_timestamp = kvm_x86_ops->read_l1_tsc(v, host_tsc);
  1278. /*
  1279. * We may have to catch up the TSC to match elapsed wall clock
  1280. * time for two reasons, even if kvmclock is used.
  1281. * 1) CPU could have been running below the maximum TSC rate
  1282. * 2) Broken TSC compensation resets the base at each VCPU
  1283. * entry to avoid unknown leaps of TSC even when running
  1284. * again on the same CPU. This may cause apparent elapsed
  1285. * time to disappear, and the guest to stand still or run
  1286. * very slowly.
  1287. */
  1288. if (vcpu->tsc_catchup) {
  1289. u64 tsc = compute_guest_tsc(v, kernel_ns);
  1290. if (tsc > tsc_timestamp) {
  1291. adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
  1292. tsc_timestamp = tsc;
  1293. }
  1294. }
  1295. local_irq_restore(flags);
  1296. if (!vcpu->pv_time_enabled)
  1297. return 0;
  1298. /*
  1299. * Time as measured by the TSC may go backwards when resetting the base
  1300. * tsc_timestamp. The reason for this is that the TSC resolution is
  1301. * higher than the resolution of the other clock scales. Thus, many
  1302. * possible measurments of the TSC correspond to one measurement of any
  1303. * other clock, and so a spread of values is possible. This is not a
  1304. * problem for the computation of the nanosecond clock; with TSC rates
  1305. * around 1GHZ, there can only be a few cycles which correspond to one
  1306. * nanosecond value, and any path through this code will inevitably
  1307. * take longer than that. However, with the kernel_ns value itself,
  1308. * the precision may be much lower, down to HZ granularity. If the
  1309. * first sampling of TSC against kernel_ns ends in the low part of the
  1310. * range, and the second in the high end of the range, we can get:
  1311. *
  1312. * (TSC - offset_low) * S + kns_old > (TSC - offset_high) * S + kns_new
  1313. *
  1314. * As the sampling errors potentially range in the thousands of cycles,
  1315. * it is possible such a time value has already been observed by the
  1316. * guest. To protect against this, we must compute the system time as
  1317. * observed by the guest and ensure the new system time is greater.
  1318. */
  1319. max_kernel_ns = 0;
  1320. if (vcpu->hv_clock.tsc_timestamp) {
  1321. max_kernel_ns = vcpu->last_guest_tsc -
  1322. vcpu->hv_clock.tsc_timestamp;
  1323. max_kernel_ns = pvclock_scale_delta(max_kernel_ns,
  1324. vcpu->hv_clock.tsc_to_system_mul,
  1325. vcpu->hv_clock.tsc_shift);
  1326. max_kernel_ns += vcpu->last_kernel_ns;
  1327. }
  1328. if (unlikely(vcpu->hw_tsc_khz != this_tsc_khz)) {
  1329. kvm_get_time_scale(NSEC_PER_SEC / 1000, this_tsc_khz,
  1330. &vcpu->hv_clock.tsc_shift,
  1331. &vcpu->hv_clock.tsc_to_system_mul);
  1332. vcpu->hw_tsc_khz = this_tsc_khz;
  1333. }
  1334. /* with a master <monotonic time, tsc value> tuple,
  1335. * pvclock clock reads always increase at the (scaled) rate
  1336. * of guest TSC - no need to deal with sampling errors.
  1337. */
  1338. if (!use_master_clock) {
  1339. if (max_kernel_ns > kernel_ns)
  1340. kernel_ns = max_kernel_ns;
  1341. }
  1342. /* With all the info we got, fill in the values */
  1343. vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
  1344. vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
  1345. vcpu->last_kernel_ns = kernel_ns;
  1346. vcpu->last_guest_tsc = tsc_timestamp;
  1347. /*
  1348. * The interface expects us to write an even number signaling that the
  1349. * update is finished. Since the guest won't see the intermediate
  1350. * state, we just increase by 2 at the end.
  1351. */
  1352. vcpu->hv_clock.version += 2;
  1353. if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
  1354. &guest_hv_clock, sizeof(guest_hv_clock))))
  1355. return 0;
  1356. /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
  1357. pvclock_flags = (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);
  1358. if (vcpu->pvclock_set_guest_stopped_request) {
  1359. pvclock_flags |= PVCLOCK_GUEST_STOPPED;
  1360. vcpu->pvclock_set_guest_stopped_request = false;
  1361. }
  1362. /* If the host uses TSC clocksource, then it is stable */
  1363. if (use_master_clock)
  1364. pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;
  1365. vcpu->hv_clock.flags = pvclock_flags;
  1366. kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
  1367. &vcpu->hv_clock,
  1368. sizeof(vcpu->hv_clock));
  1369. return 0;
  1370. }
  1371. /*
  1372. * kvmclock updates which are isolated to a given vcpu, such as
  1373. * vcpu->cpu migration, should not allow system_timestamp from
  1374. * the rest of the vcpus to remain static. Otherwise ntp frequency
  1375. * correction applies to one vcpu's system_timestamp but not
  1376. * the others.
  1377. *
  1378. * So in those cases, request a kvmclock update for all vcpus.
  1379. * The worst case for a remote vcpu to update its kvmclock
  1380. * is then bounded by maximum nohz sleep latency.
  1381. */
  1382. static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
  1383. {
  1384. int i;
  1385. struct kvm *kvm = v->kvm;
  1386. struct kvm_vcpu *vcpu;
  1387. kvm_for_each_vcpu(i, vcpu, kvm) {
  1388. set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
  1389. kvm_vcpu_kick(vcpu);
  1390. }
  1391. }
  1392. static bool msr_mtrr_valid(unsigned msr)
  1393. {
  1394. switch (msr) {
  1395. case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
  1396. case MSR_MTRRfix64K_00000:
  1397. case MSR_MTRRfix16K_80000:
  1398. case MSR_MTRRfix16K_A0000:
  1399. case MSR_MTRRfix4K_C0000:
  1400. case MSR_MTRRfix4K_C8000:
  1401. case MSR_MTRRfix4K_D0000:
  1402. case MSR_MTRRfix4K_D8000:
  1403. case MSR_MTRRfix4K_E0000:
  1404. case MSR_MTRRfix4K_E8000:
  1405. case MSR_MTRRfix4K_F0000:
  1406. case MSR_MTRRfix4K_F8000:
  1407. case MSR_MTRRdefType:
  1408. case MSR_IA32_CR_PAT:
  1409. return true;
  1410. case 0x2f8:
  1411. return true;
  1412. }
  1413. return false;
  1414. }
  1415. static bool valid_pat_type(unsigned t)
  1416. {
  1417. return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
  1418. }
  1419. static bool valid_mtrr_type(unsigned t)
  1420. {
  1421. return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
  1422. }
  1423. static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1424. {
  1425. int i;
  1426. if (!msr_mtrr_valid(msr))
  1427. return false;
  1428. if (msr == MSR_IA32_CR_PAT) {
  1429. for (i = 0; i < 8; i++)
  1430. if (!valid_pat_type((data >> (i * 8)) & 0xff))
  1431. return false;
  1432. return true;
  1433. } else if (msr == MSR_MTRRdefType) {
  1434. if (data & ~0xcff)
  1435. return false;
  1436. return valid_mtrr_type(data & 0xff);
  1437. } else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
  1438. for (i = 0; i < 8 ; i++)
  1439. if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
  1440. return false;
  1441. return true;
  1442. }
  1443. /* variable MTRRs */
  1444. return valid_mtrr_type(data & 0xff);
  1445. }
  1446. static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1447. {
  1448. u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
  1449. if (!mtrr_valid(vcpu, msr, data))
  1450. return 1;
  1451. if (msr == MSR_MTRRdefType) {
  1452. vcpu->arch.mtrr_state.def_type = data;
  1453. vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
  1454. } else if (msr == MSR_MTRRfix64K_00000)
  1455. p[0] = data;
  1456. else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
  1457. p[1 + msr - MSR_MTRRfix16K_80000] = data;
  1458. else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
  1459. p[3 + msr - MSR_MTRRfix4K_C0000] = data;
  1460. else if (msr == MSR_IA32_CR_PAT)
  1461. vcpu->arch.pat = data;
  1462. else { /* Variable MTRRs */
  1463. int idx, is_mtrr_mask;
  1464. u64 *pt;
  1465. idx = (msr - 0x200) / 2;
  1466. is_mtrr_mask = msr - 0x200 - 2 * idx;
  1467. if (!is_mtrr_mask)
  1468. pt =
  1469. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
  1470. else
  1471. pt =
  1472. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
  1473. *pt = data;
  1474. }
  1475. kvm_mmu_reset_context(vcpu);
  1476. return 0;
  1477. }
  1478. static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1479. {
  1480. u64 mcg_cap = vcpu->arch.mcg_cap;
  1481. unsigned bank_num = mcg_cap & 0xff;
  1482. switch (msr) {
  1483. case MSR_IA32_MCG_STATUS:
  1484. vcpu->arch.mcg_status = data;
  1485. break;
  1486. case MSR_IA32_MCG_CTL:
  1487. if (!(mcg_cap & MCG_CTL_P))
  1488. return 1;
  1489. if (data != 0 && data != ~(u64)0)
  1490. return -1;
  1491. vcpu->arch.mcg_ctl = data;
  1492. break;
  1493. default:
  1494. if (msr >= MSR_IA32_MC0_CTL &&
  1495. msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
  1496. u32 offset = msr - MSR_IA32_MC0_CTL;
  1497. /* only 0 or all 1s can be written to IA32_MCi_CTL
  1498. * some Linux kernels though clear bit 10 in bank 4 to
  1499. * workaround a BIOS/GART TBL issue on AMD K8s, ignore
  1500. * this to avoid an uncatched #GP in the guest
  1501. */
  1502. if ((offset & 0x3) == 0 &&
  1503. data != 0 && (data | (1 << 10)) != ~(u64)0)
  1504. return -1;
  1505. vcpu->arch.mce_banks[offset] = data;
  1506. break;
  1507. }
  1508. return 1;
  1509. }
  1510. return 0;
  1511. }
  1512. static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
  1513. {
  1514. struct kvm *kvm = vcpu->kvm;
  1515. int lm = is_long_mode(vcpu);
  1516. u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
  1517. : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
  1518. u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
  1519. : kvm->arch.xen_hvm_config.blob_size_32;
  1520. u32 page_num = data & ~PAGE_MASK;
  1521. u64 page_addr = data & PAGE_MASK;
  1522. u8 *page;
  1523. int r;
  1524. r = -E2BIG;
  1525. if (page_num >= blob_size)
  1526. goto out;
  1527. r = -ENOMEM;
  1528. page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
  1529. if (IS_ERR(page)) {
  1530. r = PTR_ERR(page);
  1531. goto out;
  1532. }
  1533. if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
  1534. goto out_free;
  1535. r = 0;
  1536. out_free:
  1537. kfree(page);
  1538. out:
  1539. return r;
  1540. }
  1541. static bool kvm_hv_hypercall_enabled(struct kvm *kvm)
  1542. {
  1543. return kvm->arch.hv_hypercall & HV_X64_MSR_HYPERCALL_ENABLE;
  1544. }
  1545. static bool kvm_hv_msr_partition_wide(u32 msr)
  1546. {
  1547. bool r = false;
  1548. switch (msr) {
  1549. case HV_X64_MSR_GUEST_OS_ID:
  1550. case HV_X64_MSR_HYPERCALL:
  1551. r = true;
  1552. break;
  1553. }
  1554. return r;
  1555. }
  1556. static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1557. {
  1558. struct kvm *kvm = vcpu->kvm;
  1559. switch (msr) {
  1560. case HV_X64_MSR_GUEST_OS_ID:
  1561. kvm->arch.hv_guest_os_id = data;
  1562. /* setting guest os id to zero disables hypercall page */
  1563. if (!kvm->arch.hv_guest_os_id)
  1564. kvm->arch.hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
  1565. break;
  1566. case HV_X64_MSR_HYPERCALL: {
  1567. u64 gfn;
  1568. unsigned long addr;
  1569. u8 instructions[4];
  1570. /* if guest os id is not set hypercall should remain disabled */
  1571. if (!kvm->arch.hv_guest_os_id)
  1572. break;
  1573. if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) {
  1574. kvm->arch.hv_hypercall = data;
  1575. break;
  1576. }
  1577. gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;
  1578. addr = gfn_to_hva(kvm, gfn);
  1579. if (kvm_is_error_hva(addr))
  1580. return 1;
  1581. kvm_x86_ops->patch_hypercall(vcpu, instructions);
  1582. ((unsigned char *)instructions)[3] = 0xc3; /* ret */
  1583. if (__copy_to_user((void __user *)addr, instructions, 4))
  1584. return 1;
  1585. kvm->arch.hv_hypercall = data;
  1586. break;
  1587. }
  1588. default:
  1589. vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
  1590. "data 0x%llx\n", msr, data);
  1591. return 1;
  1592. }
  1593. return 0;
  1594. }
  1595. static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1596. {
  1597. switch (msr) {
  1598. case HV_X64_MSR_APIC_ASSIST_PAGE: {
  1599. unsigned long addr;
  1600. if (!(data & HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE)) {
  1601. vcpu->arch.hv_vapic = data;
  1602. break;
  1603. }
  1604. addr = gfn_to_hva(vcpu->kvm, data >>
  1605. HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT);
  1606. if (kvm_is_error_hva(addr))
  1607. return 1;
  1608. if (__clear_user((void __user *)addr, PAGE_SIZE))
  1609. return 1;
  1610. vcpu->arch.hv_vapic = data;
  1611. break;
  1612. }
  1613. case HV_X64_MSR_EOI:
  1614. return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data);
  1615. case HV_X64_MSR_ICR:
  1616. return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data);
  1617. case HV_X64_MSR_TPR:
  1618. return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data);
  1619. default:
  1620. vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
  1621. "data 0x%llx\n", msr, data);
  1622. return 1;
  1623. }
  1624. return 0;
  1625. }
  1626. static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
  1627. {
  1628. gpa_t gpa = data & ~0x3f;
  1629. /* Bits 2:5 are reserved, Should be zero */
  1630. if (data & 0x3c)
  1631. return 1;
  1632. vcpu->arch.apf.msr_val = data;
  1633. if (!(data & KVM_ASYNC_PF_ENABLED)) {
  1634. kvm_clear_async_pf_completion_queue(vcpu);
  1635. kvm_async_pf_hash_reset(vcpu);
  1636. return 0;
  1637. }
  1638. if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
  1639. sizeof(u32)))
  1640. return 1;
  1641. vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
  1642. kvm_async_pf_wakeup_all(vcpu);
  1643. return 0;
  1644. }
  1645. static void kvmclock_reset(struct kvm_vcpu *vcpu)
  1646. {
  1647. vcpu->arch.pv_time_enabled = false;
  1648. }
  1649. static void accumulate_steal_time(struct kvm_vcpu *vcpu)
  1650. {
  1651. u64 delta;
  1652. if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
  1653. return;
  1654. delta = current->sched_info.run_delay - vcpu->arch.st.last_steal;
  1655. vcpu->arch.st.last_steal = current->sched_info.run_delay;
  1656. vcpu->arch.st.accum_steal = delta;
  1657. }
  1658. static void record_steal_time(struct kvm_vcpu *vcpu)
  1659. {
  1660. if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
  1661. return;
  1662. if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
  1663. &vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
  1664. return;
  1665. vcpu->arch.st.steal.steal += vcpu->arch.st.accum_steal;
  1666. vcpu->arch.st.steal.version += 2;
  1667. vcpu->arch.st.accum_steal = 0;
  1668. kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
  1669. &vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
  1670. }
  1671. int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
  1672. {
  1673. bool pr = false;
  1674. u32 msr = msr_info->index;
  1675. u64 data = msr_info->data;
  1676. switch (msr) {
  1677. case MSR_AMD64_NB_CFG:
  1678. case MSR_IA32_UCODE_REV:
  1679. case MSR_IA32_UCODE_WRITE:
  1680. case MSR_VM_HSAVE_PA:
  1681. case MSR_AMD64_PATCH_LOADER:
  1682. case MSR_AMD64_BU_CFG2:
  1683. break;
  1684. case MSR_EFER:
  1685. return set_efer(vcpu, data);
  1686. case MSR_K7_HWCR:
  1687. data &= ~(u64)0x40; /* ignore flush filter disable */
  1688. data &= ~(u64)0x100; /* ignore ignne emulation enable */
  1689. data &= ~(u64)0x8; /* ignore TLB cache disable */
  1690. if (data != 0) {
  1691. vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
  1692. data);
  1693. return 1;
  1694. }
  1695. break;
  1696. case MSR_FAM10H_MMIO_CONF_BASE:
  1697. if (data != 0) {
  1698. vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
  1699. "0x%llx\n", data);
  1700. return 1;
  1701. }
  1702. break;
  1703. case MSR_IA32_DEBUGCTLMSR:
  1704. if (!data) {
  1705. /* We support the non-activated case already */
  1706. break;
  1707. } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
  1708. /* Values other than LBR and BTF are vendor-specific,
  1709. thus reserved and should throw a #GP */
  1710. return 1;
  1711. }
  1712. vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
  1713. __func__, data);
  1714. break;
  1715. case 0x200 ... 0x2ff:
  1716. return set_msr_mtrr(vcpu, msr, data);
  1717. case MSR_IA32_APICBASE:
  1718. kvm_set_apic_base(vcpu, data);
  1719. break;
  1720. case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
  1721. return kvm_x2apic_msr_write(vcpu, msr, data);
  1722. case MSR_IA32_TSCDEADLINE:
  1723. kvm_set_lapic_tscdeadline_msr(vcpu, data);
  1724. break;
  1725. case MSR_IA32_TSC_ADJUST:
  1726. if (guest_cpuid_has_tsc_adjust(vcpu)) {
  1727. if (!msr_info->host_initiated) {
  1728. u64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
  1729. kvm_x86_ops->adjust_tsc_offset(vcpu, adj, true);
  1730. }
  1731. vcpu->arch.ia32_tsc_adjust_msr = data;
  1732. }
  1733. break;
  1734. case MSR_IA32_MISC_ENABLE:
  1735. vcpu->arch.ia32_misc_enable_msr = data;
  1736. break;
  1737. case MSR_KVM_WALL_CLOCK_NEW:
  1738. case MSR_KVM_WALL_CLOCK:
  1739. vcpu->kvm->arch.wall_clock = data;
  1740. kvm_write_wall_clock(vcpu->kvm, data);
  1741. break;
  1742. case MSR_KVM_SYSTEM_TIME_NEW:
  1743. case MSR_KVM_SYSTEM_TIME: {
  1744. u64 gpa_offset;
  1745. kvmclock_reset(vcpu);
  1746. vcpu->arch.time = data;
  1747. kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
  1748. /* we verify if the enable bit is set... */
  1749. if (!(data & 1))
  1750. break;
  1751. gpa_offset = data & ~(PAGE_MASK | 1);
  1752. if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
  1753. &vcpu->arch.pv_time, data & ~1ULL,
  1754. sizeof(struct pvclock_vcpu_time_info)))
  1755. vcpu->arch.pv_time_enabled = false;
  1756. else
  1757. vcpu->arch.pv_time_enabled = true;
  1758. break;
  1759. }
  1760. case MSR_KVM_ASYNC_PF_EN:
  1761. if (kvm_pv_enable_async_pf(vcpu, data))
  1762. return 1;
  1763. break;
  1764. case MSR_KVM_STEAL_TIME:
  1765. if (unlikely(!sched_info_on()))
  1766. return 1;
  1767. if (data & KVM_STEAL_RESERVED_MASK)
  1768. return 1;
  1769. if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
  1770. data & KVM_STEAL_VALID_BITS,
  1771. sizeof(struct kvm_steal_time)))
  1772. return 1;
  1773. vcpu->arch.st.msr_val = data;
  1774. if (!(data & KVM_MSR_ENABLED))
  1775. break;
  1776. vcpu->arch.st.last_steal = current->sched_info.run_delay;
  1777. preempt_disable();
  1778. accumulate_steal_time(vcpu);
  1779. preempt_enable();
  1780. kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
  1781. break;
  1782. case MSR_KVM_PV_EOI_EN:
  1783. if (kvm_lapic_enable_pv_eoi(vcpu, data))
  1784. return 1;
  1785. break;
  1786. case MSR_IA32_MCG_CTL:
  1787. case MSR_IA32_MCG_STATUS:
  1788. case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
  1789. return set_msr_mce(vcpu, msr, data);
  1790. /* Performance counters are not protected by a CPUID bit,
  1791. * so we should check all of them in the generic path for the sake of
  1792. * cross vendor migration.
  1793. * Writing a zero into the event select MSRs disables them,
  1794. * which we perfectly emulate ;-). Any other value should be at least
  1795. * reported, some guests depend on them.
  1796. */
  1797. case MSR_K7_EVNTSEL0:
  1798. case MSR_K7_EVNTSEL1:
  1799. case MSR_K7_EVNTSEL2:
  1800. case MSR_K7_EVNTSEL3:
  1801. if (data != 0)
  1802. vcpu_unimpl(vcpu, "unimplemented perfctr wrmsr: "
  1803. "0x%x data 0x%llx\n", msr, data);
  1804. break;
  1805. /* at least RHEL 4 unconditionally writes to the perfctr registers,
  1806. * so we ignore writes to make it happy.
  1807. */
  1808. case MSR_K7_PERFCTR0:
  1809. case MSR_K7_PERFCTR1:
  1810. case MSR_K7_PERFCTR2:
  1811. case MSR_K7_PERFCTR3:
  1812. vcpu_unimpl(vcpu, "unimplemented perfctr wrmsr: "
  1813. "0x%x data 0x%llx\n", msr, data);
  1814. break;
  1815. case MSR_P6_PERFCTR0:
  1816. case MSR_P6_PERFCTR1:
  1817. pr = true;
  1818. case MSR_P6_EVNTSEL0:
  1819. case MSR_P6_EVNTSEL1:
  1820. if (kvm_pmu_msr(vcpu, msr))
  1821. return kvm_pmu_set_msr(vcpu, msr_info);
  1822. if (pr || data != 0)
  1823. vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
  1824. "0x%x data 0x%llx\n", msr, data);
  1825. break;
  1826. case MSR_K7_CLK_CTL:
  1827. /*
  1828. * Ignore all writes to this no longer documented MSR.
  1829. * Writes are only relevant for old K7 processors,
  1830. * all pre-dating SVM, but a recommended workaround from
  1831. * AMD for these chips. It is possible to specify the
  1832. * affected processor models on the command line, hence
  1833. * the need to ignore the workaround.
  1834. */
  1835. break;
  1836. case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
  1837. if (kvm_hv_msr_partition_wide(msr)) {
  1838. int r;
  1839. mutex_lock(&vcpu->kvm->lock);
  1840. r = set_msr_hyperv_pw(vcpu, msr, data);
  1841. mutex_unlock(&vcpu->kvm->lock);
  1842. return r;
  1843. } else
  1844. return set_msr_hyperv(vcpu, msr, data);
  1845. break;
  1846. case MSR_IA32_BBL_CR_CTL3:
  1847. /* Drop writes to this legacy MSR -- see rdmsr
  1848. * counterpart for further detail.
  1849. */
  1850. vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", msr, data);
  1851. break;
  1852. case MSR_AMD64_OSVW_ID_LENGTH:
  1853. if (!guest_cpuid_has_osvw(vcpu))
  1854. return 1;
  1855. vcpu->arch.osvw.length = data;
  1856. break;
  1857. case MSR_AMD64_OSVW_STATUS:
  1858. if (!guest_cpuid_has_osvw(vcpu))
  1859. return 1;
  1860. vcpu->arch.osvw.status = data;
  1861. break;
  1862. default:
  1863. if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
  1864. return xen_hvm_config(vcpu, data);
  1865. if (kvm_pmu_msr(vcpu, msr))
  1866. return kvm_pmu_set_msr(vcpu, msr_info);
  1867. if (!ignore_msrs) {
  1868. vcpu_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
  1869. msr, data);
  1870. return 1;
  1871. } else {
  1872. vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
  1873. msr, data);
  1874. break;
  1875. }
  1876. }
  1877. return 0;
  1878. }
  1879. EXPORT_SYMBOL_GPL(kvm_set_msr_common);
  1880. /*
  1881. * Reads an msr value (of 'msr_index') into 'pdata'.
  1882. * Returns 0 on success, non-0 otherwise.
  1883. * Assumes vcpu_load() was already called.
  1884. */
  1885. int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
  1886. {
  1887. return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
  1888. }
  1889. static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1890. {
  1891. u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
  1892. if (!msr_mtrr_valid(msr))
  1893. return 1;
  1894. if (msr == MSR_MTRRdefType)
  1895. *pdata = vcpu->arch.mtrr_state.def_type +
  1896. (vcpu->arch.mtrr_state.enabled << 10);
  1897. else if (msr == MSR_MTRRfix64K_00000)
  1898. *pdata = p[0];
  1899. else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
  1900. *pdata = p[1 + msr - MSR_MTRRfix16K_80000];
  1901. else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
  1902. *pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
  1903. else if (msr == MSR_IA32_CR_PAT)
  1904. *pdata = vcpu->arch.pat;
  1905. else { /* Variable MTRRs */
  1906. int idx, is_mtrr_mask;
  1907. u64 *pt;
  1908. idx = (msr - 0x200) / 2;
  1909. is_mtrr_mask = msr - 0x200 - 2 * idx;
  1910. if (!is_mtrr_mask)
  1911. pt =
  1912. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
  1913. else
  1914. pt =
  1915. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
  1916. *pdata = *pt;
  1917. }
  1918. return 0;
  1919. }
  1920. static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1921. {
  1922. u64 data;
  1923. u64 mcg_cap = vcpu->arch.mcg_cap;
  1924. unsigned bank_num = mcg_cap & 0xff;
  1925. switch (msr) {
  1926. case MSR_IA32_P5_MC_ADDR:
  1927. case MSR_IA32_P5_MC_TYPE:
  1928. data = 0;
  1929. break;
  1930. case MSR_IA32_MCG_CAP:
  1931. data = vcpu->arch.mcg_cap;
  1932. break;
  1933. case MSR_IA32_MCG_CTL:
  1934. if (!(mcg_cap & MCG_CTL_P))
  1935. return 1;
  1936. data = vcpu->arch.mcg_ctl;
  1937. break;
  1938. case MSR_IA32_MCG_STATUS:
  1939. data = vcpu->arch.mcg_status;
  1940. break;
  1941. default:
  1942. if (msr >= MSR_IA32_MC0_CTL &&
  1943. msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
  1944. u32 offset = msr - MSR_IA32_MC0_CTL;
  1945. data = vcpu->arch.mce_banks[offset];
  1946. break;
  1947. }
  1948. return 1;
  1949. }
  1950. *pdata = data;
  1951. return 0;
  1952. }
  1953. static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1954. {
  1955. u64 data = 0;
  1956. struct kvm *kvm = vcpu->kvm;
  1957. switch (msr) {
  1958. case HV_X64_MSR_GUEST_OS_ID:
  1959. data = kvm->arch.hv_guest_os_id;
  1960. break;
  1961. case HV_X64_MSR_HYPERCALL:
  1962. data = kvm->arch.hv_hypercall;
  1963. break;
  1964. default:
  1965. vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
  1966. return 1;
  1967. }
  1968. *pdata = data;
  1969. return 0;
  1970. }
  1971. static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1972. {
  1973. u64 data = 0;
  1974. switch (msr) {
  1975. case HV_X64_MSR_VP_INDEX: {
  1976. int r;
  1977. struct kvm_vcpu *v;
  1978. kvm_for_each_vcpu(r, v, vcpu->kvm)
  1979. if (v == vcpu)
  1980. data = r;
  1981. break;
  1982. }
  1983. case HV_X64_MSR_EOI:
  1984. return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata);
  1985. case HV_X64_MSR_ICR:
  1986. return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
  1987. case HV_X64_MSR_TPR:
  1988. return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
  1989. case HV_X64_MSR_APIC_ASSIST_PAGE:
  1990. data = vcpu->arch.hv_vapic;
  1991. break;
  1992. default:
  1993. vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
  1994. return 1;
  1995. }
  1996. *pdata = data;
  1997. return 0;
  1998. }
  1999. int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  2000. {
  2001. u64 data;
  2002. switch (msr) {
  2003. case MSR_IA32_PLATFORM_ID:
  2004. case MSR_IA32_EBL_CR_POWERON:
  2005. case MSR_IA32_DEBUGCTLMSR:
  2006. case MSR_IA32_LASTBRANCHFROMIP:
  2007. case MSR_IA32_LASTBRANCHTOIP:
  2008. case MSR_IA32_LASTINTFROMIP:
  2009. case MSR_IA32_LASTINTTOIP:
  2010. case MSR_K8_SYSCFG:
  2011. case MSR_K7_HWCR:
  2012. case MSR_VM_HSAVE_PA:
  2013. case MSR_K7_EVNTSEL0:
  2014. case MSR_K7_PERFCTR0:
  2015. case MSR_K8_INT_PENDING_MSG:
  2016. case MSR_AMD64_NB_CFG:
  2017. case MSR_FAM10H_MMIO_CONF_BASE:
  2018. case MSR_AMD64_BU_CFG2:
  2019. data = 0;
  2020. break;
  2021. case MSR_P6_PERFCTR0:
  2022. case MSR_P6_PERFCTR1:
  2023. case MSR_P6_EVNTSEL0:
  2024. case MSR_P6_EVNTSEL1:
  2025. if (kvm_pmu_msr(vcpu, msr))
  2026. return kvm_pmu_get_msr(vcpu, msr, pdata);
  2027. data = 0;
  2028. break;
  2029. case MSR_IA32_UCODE_REV:
  2030. data = 0x100000000ULL;
  2031. break;
  2032. case MSR_MTRRcap:
  2033. data = 0x500 | KVM_NR_VAR_MTRR;
  2034. break;
  2035. case 0x200 ... 0x2ff:
  2036. return get_msr_mtrr(vcpu, msr, pdata);
  2037. case 0xcd: /* fsb frequency */
  2038. data = 3;
  2039. break;
  2040. /*
  2041. * MSR_EBC_FREQUENCY_ID
  2042. * Conservative value valid for even the basic CPU models.
  2043. * Models 0,1: 000 in bits 23:21 indicating a bus speed of
  2044. * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
  2045. * and 266MHz for model 3, or 4. Set Core Clock
  2046. * Frequency to System Bus Frequency Ratio to 1 (bits
  2047. * 31:24) even though these are only valid for CPU
  2048. * models > 2, however guests may end up dividing or
  2049. * multiplying by zero otherwise.
  2050. */
  2051. case MSR_EBC_FREQUENCY_ID:
  2052. data = 1 << 24;
  2053. break;
  2054. case MSR_IA32_APICBASE:
  2055. data = kvm_get_apic_base(vcpu);
  2056. break;
  2057. case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
  2058. return kvm_x2apic_msr_read(vcpu, msr, pdata);
  2059. break;
  2060. case MSR_IA32_TSCDEADLINE:
  2061. data = kvm_get_lapic_tscdeadline_msr(vcpu);
  2062. break;
  2063. case MSR_IA32_TSC_ADJUST:
  2064. data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
  2065. break;
  2066. case MSR_IA32_MISC_ENABLE:
  2067. data = vcpu->arch.ia32_misc_enable_msr;
  2068. break;
  2069. case MSR_IA32_PERF_STATUS:
  2070. /* TSC increment by tick */
  2071. data = 1000ULL;
  2072. /* CPU multiplier */
  2073. data |= (((uint64_t)4ULL) << 40);
  2074. break;
  2075. case MSR_EFER:
  2076. data = vcpu->arch.efer;
  2077. break;
  2078. case MSR_KVM_WALL_CLOCK:
  2079. case MSR_KVM_WALL_CLOCK_NEW:
  2080. data = vcpu->kvm->arch.wall_clock;
  2081. break;
  2082. case MSR_KVM_SYSTEM_TIME:
  2083. case MSR_KVM_SYSTEM_TIME_NEW:
  2084. data = vcpu->arch.time;
  2085. break;
  2086. case MSR_KVM_ASYNC_PF_EN:
  2087. data = vcpu->arch.apf.msr_val;
  2088. break;
  2089. case MSR_KVM_STEAL_TIME:
  2090. data = vcpu->arch.st.msr_val;
  2091. break;
  2092. case MSR_KVM_PV_EOI_EN:
  2093. data = vcpu->arch.pv_eoi.msr_val;
  2094. break;
  2095. case MSR_IA32_P5_MC_ADDR:
  2096. case MSR_IA32_P5_MC_TYPE:
  2097. case MSR_IA32_MCG_CAP:
  2098. case MSR_IA32_MCG_CTL:
  2099. case MSR_IA32_MCG_STATUS:
  2100. case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
  2101. return get_msr_mce(vcpu, msr, pdata);
  2102. case MSR_K7_CLK_CTL:
  2103. /*
  2104. * Provide expected ramp-up count for K7. All other
  2105. * are set to zero, indicating minimum divisors for
  2106. * every field.
  2107. *
  2108. * This prevents guest kernels on AMD host with CPU
  2109. * type 6, model 8 and higher from exploding due to
  2110. * the rdmsr failing.
  2111. */
  2112. data = 0x20000000;
  2113. break;
  2114. case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
  2115. if (kvm_hv_msr_partition_wide(msr)) {
  2116. int r;
  2117. mutex_lock(&vcpu->kvm->lock);
  2118. r = get_msr_hyperv_pw(vcpu, msr, pdata);
  2119. mutex_unlock(&vcpu->kvm->lock);
  2120. return r;
  2121. } else
  2122. return get_msr_hyperv(vcpu, msr, pdata);
  2123. break;
  2124. case MSR_IA32_BBL_CR_CTL3:
  2125. /* This legacy MSR exists but isn't fully documented in current
  2126. * silicon. It is however accessed by winxp in very narrow
  2127. * scenarios where it sets bit #19, itself documented as
  2128. * a "reserved" bit. Best effort attempt to source coherent
  2129. * read data here should the balance of the register be
  2130. * interpreted by the guest:
  2131. *
  2132. * L2 cache control register 3: 64GB range, 256KB size,
  2133. * enabled, latency 0x1, configured
  2134. */
  2135. data = 0xbe702111;
  2136. break;
  2137. case MSR_AMD64_OSVW_ID_LENGTH:
  2138. if (!guest_cpuid_has_osvw(vcpu))
  2139. return 1;
  2140. data = vcpu->arch.osvw.length;
  2141. break;
  2142. case MSR_AMD64_OSVW_STATUS:
  2143. if (!guest_cpuid_has_osvw(vcpu))
  2144. return 1;
  2145. data = vcpu->arch.osvw.status;
  2146. break;
  2147. default:
  2148. if (kvm_pmu_msr(vcpu, msr))
  2149. return kvm_pmu_get_msr(vcpu, msr, pdata);
  2150. if (!ignore_msrs) {
  2151. vcpu_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
  2152. return 1;
  2153. } else {
  2154. vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
  2155. data = 0;
  2156. }
  2157. break;
  2158. }
  2159. *pdata = data;
  2160. return 0;
  2161. }
  2162. EXPORT_SYMBOL_GPL(kvm_get_msr_common);
  2163. /*
  2164. * Read or write a bunch of msrs. All parameters are kernel addresses.
  2165. *
  2166. * @return number of msrs set successfully.
  2167. */
  2168. static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
  2169. struct kvm_msr_entry *entries,
  2170. int (*do_msr)(struct kvm_vcpu *vcpu,
  2171. unsigned index, u64 *data))
  2172. {
  2173. int i, idx;
  2174. idx = srcu_read_lock(&vcpu->kvm->srcu);
  2175. for (i = 0; i < msrs->nmsrs; ++i)
  2176. if (do_msr(vcpu, entries[i].index, &entries[i].data))
  2177. break;
  2178. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  2179. return i;
  2180. }
  2181. /*
  2182. * Read or write a bunch of msrs. Parameters are user addresses.
  2183. *
  2184. * @return number of msrs set successfully.
  2185. */
  2186. static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
  2187. int (*do_msr)(struct kvm_vcpu *vcpu,
  2188. unsigned index, u64 *data),
  2189. int writeback)
  2190. {
  2191. struct kvm_msrs msrs;
  2192. struct kvm_msr_entry *entries;
  2193. int r, n;
  2194. unsigned size;
  2195. r = -EFAULT;
  2196. if (copy_from_user(&msrs, user_msrs, sizeof msrs))
  2197. goto out;
  2198. r = -E2BIG;
  2199. if (msrs.nmsrs >= MAX_IO_MSRS)
  2200. goto out;
  2201. size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
  2202. entries = memdup_user(user_msrs->entries, size);
  2203. if (IS_ERR(entries)) {
  2204. r = PTR_ERR(entries);
  2205. goto out;
  2206. }
  2207. r = n = __msr_io(vcpu, &msrs, entries, do_msr);
  2208. if (r < 0)
  2209. goto out_free;
  2210. r = -EFAULT;
  2211. if (writeback && copy_to_user(user_msrs->entries, entries, size))
  2212. goto out_free;
  2213. r = n;
  2214. out_free:
  2215. kfree(entries);
  2216. out:
  2217. return r;
  2218. }
  2219. int kvm_dev_ioctl_check_extension(long ext)
  2220. {
  2221. int r;
  2222. switch (ext) {
  2223. case KVM_CAP_IRQCHIP:
  2224. case KVM_CAP_HLT:
  2225. case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
  2226. case KVM_CAP_SET_TSS_ADDR:
  2227. case KVM_CAP_EXT_CPUID:
  2228. case KVM_CAP_CLOCKSOURCE:
  2229. case KVM_CAP_PIT:
  2230. case KVM_CAP_NOP_IO_DELAY:
  2231. case KVM_CAP_MP_STATE:
  2232. case KVM_CAP_SYNC_MMU:
  2233. case KVM_CAP_USER_NMI:
  2234. case KVM_CAP_REINJECT_CONTROL:
  2235. case KVM_CAP_IRQ_INJECT_STATUS:
  2236. case KVM_CAP_IRQFD:
  2237. case KVM_CAP_IOEVENTFD:
  2238. case KVM_CAP_PIT2:
  2239. case KVM_CAP_PIT_STATE2:
  2240. case KVM_CAP_SET_IDENTITY_MAP_ADDR:
  2241. case KVM_CAP_XEN_HVM:
  2242. case KVM_CAP_ADJUST_CLOCK:
  2243. case KVM_CAP_VCPU_EVENTS:
  2244. case KVM_CAP_HYPERV:
  2245. case KVM_CAP_HYPERV_VAPIC:
  2246. case KVM_CAP_HYPERV_SPIN:
  2247. case KVM_CAP_PCI_SEGMENT:
  2248. case KVM_CAP_DEBUGREGS:
  2249. case KVM_CAP_X86_ROBUST_SINGLESTEP:
  2250. case KVM_CAP_XSAVE:
  2251. case KVM_CAP_ASYNC_PF:
  2252. case KVM_CAP_GET_TSC_KHZ:
  2253. case KVM_CAP_KVMCLOCK_CTRL:
  2254. case KVM_CAP_READONLY_MEM:
  2255. #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
  2256. case KVM_CAP_ASSIGN_DEV_IRQ:
  2257. case KVM_CAP_PCI_2_3:
  2258. #endif
  2259. r = 1;
  2260. break;
  2261. case KVM_CAP_COALESCED_MMIO:
  2262. r = KVM_COALESCED_MMIO_PAGE_OFFSET;
  2263. break;
  2264. case KVM_CAP_VAPIC:
  2265. r = !kvm_x86_ops->cpu_has_accelerated_tpr();
  2266. break;
  2267. case KVM_CAP_NR_VCPUS:
  2268. r = KVM_SOFT_MAX_VCPUS;
  2269. break;
  2270. case KVM_CAP_MAX_VCPUS:
  2271. r = KVM_MAX_VCPUS;
  2272. break;
  2273. case KVM_CAP_NR_MEMSLOTS:
  2274. r = KVM_USER_MEM_SLOTS;
  2275. break;
  2276. case KVM_CAP_PV_MMU: /* obsolete */
  2277. r = 0;
  2278. break;
  2279. #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
  2280. case KVM_CAP_IOMMU:
  2281. r = iommu_present(&pci_bus_type);
  2282. break;
  2283. #endif
  2284. case KVM_CAP_MCE:
  2285. r = KVM_MAX_MCE_BANKS;
  2286. break;
  2287. case KVM_CAP_XCRS:
  2288. r = cpu_has_xsave;
  2289. break;
  2290. case KVM_CAP_TSC_CONTROL:
  2291. r = kvm_has_tsc_control;
  2292. break;
  2293. case KVM_CAP_TSC_DEADLINE_TIMER:
  2294. r = boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER);
  2295. break;
  2296. default:
  2297. r = 0;
  2298. break;
  2299. }
  2300. return r;
  2301. }
  2302. long kvm_arch_dev_ioctl(struct file *filp,
  2303. unsigned int ioctl, unsigned long arg)
  2304. {
  2305. void __user *argp = (void __user *)arg;
  2306. long r;
  2307. switch (ioctl) {
  2308. case KVM_GET_MSR_INDEX_LIST: {
  2309. struct kvm_msr_list __user *user_msr_list = argp;
  2310. struct kvm_msr_list msr_list;
  2311. unsigned n;
  2312. r = -EFAULT;
  2313. if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
  2314. goto out;
  2315. n = msr_list.nmsrs;
  2316. msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
  2317. if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
  2318. goto out;
  2319. r = -E2BIG;
  2320. if (n < msr_list.nmsrs)
  2321. goto out;
  2322. r = -EFAULT;
  2323. if (copy_to_user(user_msr_list->indices, &msrs_to_save,
  2324. num_msrs_to_save * sizeof(u32)))
  2325. goto out;
  2326. if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
  2327. &emulated_msrs,
  2328. ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
  2329. goto out;
  2330. r = 0;
  2331. break;
  2332. }
  2333. case KVM_GET_SUPPORTED_CPUID: {
  2334. struct kvm_cpuid2 __user *cpuid_arg = argp;
  2335. struct kvm_cpuid2 cpuid;
  2336. r = -EFAULT;
  2337. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  2338. goto out;
  2339. r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
  2340. cpuid_arg->entries);
  2341. if (r)
  2342. goto out;
  2343. r = -EFAULT;
  2344. if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
  2345. goto out;
  2346. r = 0;
  2347. break;
  2348. }
  2349. case KVM_X86_GET_MCE_CAP_SUPPORTED: {
  2350. u64 mce_cap;
  2351. mce_cap = KVM_MCE_CAP_SUPPORTED;
  2352. r = -EFAULT;
  2353. if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
  2354. goto out;
  2355. r = 0;
  2356. break;
  2357. }
  2358. default:
  2359. r = -EINVAL;
  2360. }
  2361. out:
  2362. return r;
  2363. }
  2364. static void wbinvd_ipi(void *garbage)
  2365. {
  2366. wbinvd();
  2367. }
  2368. static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
  2369. {
  2370. return vcpu->kvm->arch.iommu_domain &&
  2371. !(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY);
  2372. }
  2373. void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  2374. {
  2375. /* Address WBINVD may be executed by guest */
  2376. if (need_emulate_wbinvd(vcpu)) {
  2377. if (kvm_x86_ops->has_wbinvd_exit())
  2378. cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
  2379. else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
  2380. smp_call_function_single(vcpu->cpu,
  2381. wbinvd_ipi, NULL, 1);
  2382. }
  2383. kvm_x86_ops->vcpu_load(vcpu, cpu);
  2384. /* Apply any externally detected TSC adjustments (due to suspend) */
  2385. if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
  2386. adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
  2387. vcpu->arch.tsc_offset_adjustment = 0;
  2388. set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
  2389. }
  2390. if (unlikely(vcpu->cpu != cpu) || check_tsc_unstable()) {
  2391. s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
  2392. native_read_tsc() - vcpu->arch.last_host_tsc;
  2393. if (tsc_delta < 0)
  2394. mark_tsc_unstable("KVM discovered backwards TSC");
  2395. if (check_tsc_unstable()) {
  2396. u64 offset = kvm_x86_ops->compute_tsc_offset(vcpu,
  2397. vcpu->arch.last_guest_tsc);
  2398. kvm_x86_ops->write_tsc_offset(vcpu, offset);
  2399. vcpu->arch.tsc_catchup = 1;
  2400. }
  2401. /*
  2402. * On a host with synchronized TSC, there is no need to update
  2403. * kvmclock on vcpu->cpu migration
  2404. */
  2405. if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
  2406. kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
  2407. if (vcpu->cpu != cpu)
  2408. kvm_migrate_timers(vcpu);
  2409. vcpu->cpu = cpu;
  2410. }
  2411. accumulate_steal_time(vcpu);
  2412. kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
  2413. }
  2414. void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
  2415. {
  2416. kvm_x86_ops->vcpu_put(vcpu);
  2417. kvm_put_guest_fpu(vcpu);
  2418. vcpu->arch.last_host_tsc = native_read_tsc();
  2419. }
  2420. static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
  2421. struct kvm_lapic_state *s)
  2422. {
  2423. kvm_x86_ops->sync_pir_to_irr(vcpu);
  2424. memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
  2425. return 0;
  2426. }
  2427. static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
  2428. struct kvm_lapic_state *s)
  2429. {
  2430. kvm_apic_post_state_restore(vcpu, s);
  2431. update_cr8_intercept(vcpu);
  2432. return 0;
  2433. }
  2434. static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
  2435. struct kvm_interrupt *irq)
  2436. {
  2437. if (irq->irq >= KVM_NR_INTERRUPTS)
  2438. return -EINVAL;
  2439. if (irqchip_in_kernel(vcpu->kvm))
  2440. return -ENXIO;
  2441. kvm_queue_interrupt(vcpu, irq->irq, false);
  2442. kvm_make_request(KVM_REQ_EVENT, vcpu);
  2443. return 0;
  2444. }
  2445. static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
  2446. {
  2447. kvm_inject_nmi(vcpu);
  2448. return 0;
  2449. }
  2450. static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
  2451. struct kvm_tpr_access_ctl *tac)
  2452. {
  2453. if (tac->flags)
  2454. return -EINVAL;
  2455. vcpu->arch.tpr_access_reporting = !!tac->enabled;
  2456. return 0;
  2457. }
  2458. static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
  2459. u64 mcg_cap)
  2460. {
  2461. int r;
  2462. unsigned bank_num = mcg_cap & 0xff, bank;
  2463. r = -EINVAL;
  2464. if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
  2465. goto out;
  2466. if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
  2467. goto out;
  2468. r = 0;
  2469. vcpu->arch.mcg_cap = mcg_cap;
  2470. /* Init IA32_MCG_CTL to all 1s */
  2471. if (mcg_cap & MCG_CTL_P)
  2472. vcpu->arch.mcg_ctl = ~(u64)0;
  2473. /* Init IA32_MCi_CTL to all 1s */
  2474. for (bank = 0; bank < bank_num; bank++)
  2475. vcpu->arch.mce_banks[bank*4] = ~(u64)0;
  2476. out:
  2477. return r;
  2478. }
  2479. static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
  2480. struct kvm_x86_mce *mce)
  2481. {
  2482. u64 mcg_cap = vcpu->arch.mcg_cap;
  2483. unsigned bank_num = mcg_cap & 0xff;
  2484. u64 *banks = vcpu->arch.mce_banks;
  2485. if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
  2486. return -EINVAL;
  2487. /*
  2488. * if IA32_MCG_CTL is not all 1s, the uncorrected error
  2489. * reporting is disabled
  2490. */
  2491. if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
  2492. vcpu->arch.mcg_ctl != ~(u64)0)
  2493. return 0;
  2494. banks += 4 * mce->bank;
  2495. /*
  2496. * if IA32_MCi_CTL is not all 1s, the uncorrected error
  2497. * reporting is disabled for the bank
  2498. */
  2499. if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
  2500. return 0;
  2501. if (mce->status & MCI_STATUS_UC) {
  2502. if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
  2503. !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
  2504. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  2505. return 0;
  2506. }
  2507. if (banks[1] & MCI_STATUS_VAL)
  2508. mce->status |= MCI_STATUS_OVER;
  2509. banks[2] = mce->addr;
  2510. banks[3] = mce->misc;
  2511. vcpu->arch.mcg_status = mce->mcg_status;
  2512. banks[1] = mce->status;
  2513. kvm_queue_exception(vcpu, MC_VECTOR);
  2514. } else if (!(banks[1] & MCI_STATUS_VAL)
  2515. || !(banks[1] & MCI_STATUS_UC)) {
  2516. if (banks[1] & MCI_STATUS_VAL)
  2517. mce->status |= MCI_STATUS_OVER;
  2518. banks[2] = mce->addr;
  2519. banks[3] = mce->misc;
  2520. banks[1] = mce->status;
  2521. } else
  2522. banks[1] |= MCI_STATUS_OVER;
  2523. return 0;
  2524. }
  2525. static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
  2526. struct kvm_vcpu_events *events)
  2527. {
  2528. process_nmi(vcpu);
  2529. events->exception.injected =
  2530. vcpu->arch.exception.pending &&
  2531. !kvm_exception_is_soft(vcpu->arch.exception.nr);
  2532. events->exception.nr = vcpu->arch.exception.nr;
  2533. events->exception.has_error_code = vcpu->arch.exception.has_error_code;
  2534. events->exception.pad = 0;
  2535. events->exception.error_code = vcpu->arch.exception.error_code;
  2536. events->interrupt.injected =
  2537. vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft;
  2538. events->interrupt.nr = vcpu->arch.interrupt.nr;
  2539. events->interrupt.soft = 0;
  2540. events->interrupt.shadow =
  2541. kvm_x86_ops->get_interrupt_shadow(vcpu,
  2542. KVM_X86_SHADOW_INT_MOV_SS | KVM_X86_SHADOW_INT_STI);
  2543. events->nmi.injected = vcpu->arch.nmi_injected;
  2544. events->nmi.pending = vcpu->arch.nmi_pending != 0;
  2545. events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
  2546. events->nmi.pad = 0;
  2547. events->sipi_vector = 0; /* never valid when reporting to user space */
  2548. events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
  2549. | KVM_VCPUEVENT_VALID_SHADOW);
  2550. memset(&events->reserved, 0, sizeof(events->reserved));
  2551. }
  2552. static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
  2553. struct kvm_vcpu_events *events)
  2554. {
  2555. if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
  2556. | KVM_VCPUEVENT_VALID_SIPI_VECTOR
  2557. | KVM_VCPUEVENT_VALID_SHADOW))
  2558. return -EINVAL;
  2559. process_nmi(vcpu);
  2560. vcpu->arch.exception.pending = events->exception.injected;
  2561. vcpu->arch.exception.nr = events->exception.nr;
  2562. vcpu->arch.exception.has_error_code = events->exception.has_error_code;
  2563. vcpu->arch.exception.error_code = events->exception.error_code;
  2564. vcpu->arch.interrupt.pending = events->interrupt.injected;
  2565. vcpu->arch.interrupt.nr = events->interrupt.nr;
  2566. vcpu->arch.interrupt.soft = events->interrupt.soft;
  2567. if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
  2568. kvm_x86_ops->set_interrupt_shadow(vcpu,
  2569. events->interrupt.shadow);
  2570. vcpu->arch.nmi_injected = events->nmi.injected;
  2571. if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
  2572. vcpu->arch.nmi_pending = events->nmi.pending;
  2573. kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);
  2574. if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
  2575. kvm_vcpu_has_lapic(vcpu))
  2576. vcpu->arch.apic->sipi_vector = events->sipi_vector;
  2577. kvm_make_request(KVM_REQ_EVENT, vcpu);
  2578. return 0;
  2579. }
  2580. static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
  2581. struct kvm_debugregs *dbgregs)
  2582. {
  2583. memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
  2584. dbgregs->dr6 = vcpu->arch.dr6;
  2585. dbgregs->dr7 = vcpu->arch.dr7;
  2586. dbgregs->flags = 0;
  2587. memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
  2588. }
  2589. static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
  2590. struct kvm_debugregs *dbgregs)
  2591. {
  2592. if (dbgregs->flags)
  2593. return -EINVAL;
  2594. memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
  2595. vcpu->arch.dr6 = dbgregs->dr6;
  2596. vcpu->arch.dr7 = dbgregs->dr7;
  2597. return 0;
  2598. }
  2599. static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
  2600. struct kvm_xsave *guest_xsave)
  2601. {
  2602. if (cpu_has_xsave)
  2603. memcpy(guest_xsave->region,
  2604. &vcpu->arch.guest_fpu.state->xsave,
  2605. xstate_size);
  2606. else {
  2607. memcpy(guest_xsave->region,
  2608. &vcpu->arch.guest_fpu.state->fxsave,
  2609. sizeof(struct i387_fxsave_struct));
  2610. *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
  2611. XSTATE_FPSSE;
  2612. }
  2613. }
  2614. static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
  2615. struct kvm_xsave *guest_xsave)
  2616. {
  2617. u64 xstate_bv =
  2618. *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
  2619. if (cpu_has_xsave)
  2620. memcpy(&vcpu->arch.guest_fpu.state->xsave,
  2621. guest_xsave->region, xstate_size);
  2622. else {
  2623. if (xstate_bv & ~XSTATE_FPSSE)
  2624. return -EINVAL;
  2625. memcpy(&vcpu->arch.guest_fpu.state->fxsave,
  2626. guest_xsave->region, sizeof(struct i387_fxsave_struct));
  2627. }
  2628. return 0;
  2629. }
  2630. static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
  2631. struct kvm_xcrs *guest_xcrs)
  2632. {
  2633. if (!cpu_has_xsave) {
  2634. guest_xcrs->nr_xcrs = 0;
  2635. return;
  2636. }
  2637. guest_xcrs->nr_xcrs = 1;
  2638. guest_xcrs->flags = 0;
  2639. guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
  2640. guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
  2641. }
  2642. static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
  2643. struct kvm_xcrs *guest_xcrs)
  2644. {
  2645. int i, r = 0;
  2646. if (!cpu_has_xsave)
  2647. return -EINVAL;
  2648. if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
  2649. return -EINVAL;
  2650. for (i = 0; i < guest_xcrs->nr_xcrs; i++)
  2651. /* Only support XCR0 currently */
  2652. if (guest_xcrs->xcrs[0].xcr == XCR_XFEATURE_ENABLED_MASK) {
  2653. r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
  2654. guest_xcrs->xcrs[0].value);
  2655. break;
  2656. }
  2657. if (r)
  2658. r = -EINVAL;
  2659. return r;
  2660. }
  2661. /*
  2662. * kvm_set_guest_paused() indicates to the guest kernel that it has been
  2663. * stopped by the hypervisor. This function will be called from the host only.
  2664. * EINVAL is returned when the host attempts to set the flag for a guest that
  2665. * does not support pv clocks.
  2666. */
  2667. static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
  2668. {
  2669. if (!vcpu->arch.pv_time_enabled)
  2670. return -EINVAL;
  2671. vcpu->arch.pvclock_set_guest_stopped_request = true;
  2672. kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
  2673. return 0;
  2674. }
  2675. long kvm_arch_vcpu_ioctl(struct file *filp,
  2676. unsigned int ioctl, unsigned long arg)
  2677. {
  2678. struct kvm_vcpu *vcpu = filp->private_data;
  2679. void __user *argp = (void __user *)arg;
  2680. int r;
  2681. union {
  2682. struct kvm_lapic_state *lapic;
  2683. struct kvm_xsave *xsave;
  2684. struct kvm_xcrs *xcrs;
  2685. void *buffer;
  2686. } u;
  2687. u.buffer = NULL;
  2688. switch (ioctl) {
  2689. case KVM_GET_LAPIC: {
  2690. r = -EINVAL;
  2691. if (!vcpu->arch.apic)
  2692. goto out;
  2693. u.lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
  2694. r = -ENOMEM;
  2695. if (!u.lapic)
  2696. goto out;
  2697. r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
  2698. if (r)
  2699. goto out;
  2700. r = -EFAULT;
  2701. if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
  2702. goto out;
  2703. r = 0;
  2704. break;
  2705. }
  2706. case KVM_SET_LAPIC: {
  2707. r = -EINVAL;
  2708. if (!vcpu->arch.apic)
  2709. goto out;
  2710. u.lapic = memdup_user(argp, sizeof(*u.lapic));
  2711. if (IS_ERR(u.lapic))
  2712. return PTR_ERR(u.lapic);
  2713. r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
  2714. break;
  2715. }
  2716. case KVM_INTERRUPT: {
  2717. struct kvm_interrupt irq;
  2718. r = -EFAULT;
  2719. if (copy_from_user(&irq, argp, sizeof irq))
  2720. goto out;
  2721. r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
  2722. break;
  2723. }
  2724. case KVM_NMI: {
  2725. r = kvm_vcpu_ioctl_nmi(vcpu);
  2726. break;
  2727. }
  2728. case KVM_SET_CPUID: {
  2729. struct kvm_cpuid __user *cpuid_arg = argp;
  2730. struct kvm_cpuid cpuid;
  2731. r = -EFAULT;
  2732. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  2733. goto out;
  2734. r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
  2735. break;
  2736. }
  2737. case KVM_SET_CPUID2: {
  2738. struct kvm_cpuid2 __user *cpuid_arg = argp;
  2739. struct kvm_cpuid2 cpuid;
  2740. r = -EFAULT;
  2741. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  2742. goto out;
  2743. r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
  2744. cpuid_arg->entries);
  2745. break;
  2746. }
  2747. case KVM_GET_CPUID2: {
  2748. struct kvm_cpuid2 __user *cpuid_arg = argp;
  2749. struct kvm_cpuid2 cpuid;
  2750. r = -EFAULT;
  2751. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  2752. goto out;
  2753. r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
  2754. cpuid_arg->entries);
  2755. if (r)
  2756. goto out;
  2757. r = -EFAULT;
  2758. if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
  2759. goto out;
  2760. r = 0;
  2761. break;
  2762. }
  2763. case KVM_GET_MSRS:
  2764. r = msr_io(vcpu, argp, kvm_get_msr, 1);
  2765. break;
  2766. case KVM_SET_MSRS:
  2767. r = msr_io(vcpu, argp, do_set_msr, 0);
  2768. break;
  2769. case KVM_TPR_ACCESS_REPORTING: {
  2770. struct kvm_tpr_access_ctl tac;
  2771. r = -EFAULT;
  2772. if (copy_from_user(&tac, argp, sizeof tac))
  2773. goto out;
  2774. r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
  2775. if (r)
  2776. goto out;
  2777. r = -EFAULT;
  2778. if (copy_to_user(argp, &tac, sizeof tac))
  2779. goto out;
  2780. r = 0;
  2781. break;
  2782. };
  2783. case KVM_SET_VAPIC_ADDR: {
  2784. struct kvm_vapic_addr va;
  2785. r = -EINVAL;
  2786. if (!irqchip_in_kernel(vcpu->kvm))
  2787. goto out;
  2788. r = -EFAULT;
  2789. if (copy_from_user(&va, argp, sizeof va))
  2790. goto out;
  2791. r = 0;
  2792. kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
  2793. break;
  2794. }
  2795. case KVM_X86_SETUP_MCE: {
  2796. u64 mcg_cap;
  2797. r = -EFAULT;
  2798. if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
  2799. goto out;
  2800. r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
  2801. break;
  2802. }
  2803. case KVM_X86_SET_MCE: {
  2804. struct kvm_x86_mce mce;
  2805. r = -EFAULT;
  2806. if (copy_from_user(&mce, argp, sizeof mce))
  2807. goto out;
  2808. r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
  2809. break;
  2810. }
  2811. case KVM_GET_VCPU_EVENTS: {
  2812. struct kvm_vcpu_events events;
  2813. kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
  2814. r = -EFAULT;
  2815. if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
  2816. break;
  2817. r = 0;
  2818. break;
  2819. }
  2820. case KVM_SET_VCPU_EVENTS: {
  2821. struct kvm_vcpu_events events;
  2822. r = -EFAULT;
  2823. if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
  2824. break;
  2825. r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
  2826. break;
  2827. }
  2828. case KVM_GET_DEBUGREGS: {
  2829. struct kvm_debugregs dbgregs;
  2830. kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
  2831. r = -EFAULT;
  2832. if (copy_to_user(argp, &dbgregs,
  2833. sizeof(struct kvm_debugregs)))
  2834. break;
  2835. r = 0;
  2836. break;
  2837. }
  2838. case KVM_SET_DEBUGREGS: {
  2839. struct kvm_debugregs dbgregs;
  2840. r = -EFAULT;
  2841. if (copy_from_user(&dbgregs, argp,
  2842. sizeof(struct kvm_debugregs)))
  2843. break;
  2844. r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
  2845. break;
  2846. }
  2847. case KVM_GET_XSAVE: {
  2848. u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL);
  2849. r = -ENOMEM;
  2850. if (!u.xsave)
  2851. break;
  2852. kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
  2853. r = -EFAULT;
  2854. if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
  2855. break;
  2856. r = 0;
  2857. break;
  2858. }
  2859. case KVM_SET_XSAVE: {
  2860. u.xsave = memdup_user(argp, sizeof(*u.xsave));
  2861. if (IS_ERR(u.xsave))
  2862. return PTR_ERR(u.xsave);
  2863. r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
  2864. break;
  2865. }
  2866. case KVM_GET_XCRS: {
  2867. u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL);
  2868. r = -ENOMEM;
  2869. if (!u.xcrs)
  2870. break;
  2871. kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
  2872. r = -EFAULT;
  2873. if (copy_to_user(argp, u.xcrs,
  2874. sizeof(struct kvm_xcrs)))
  2875. break;
  2876. r = 0;
  2877. break;
  2878. }
  2879. case KVM_SET_XCRS: {
  2880. u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
  2881. if (IS_ERR(u.xcrs))
  2882. return PTR_ERR(u.xcrs);
  2883. r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
  2884. break;
  2885. }
  2886. case KVM_SET_TSC_KHZ: {
  2887. u32 user_tsc_khz;
  2888. r = -EINVAL;
  2889. user_tsc_khz = (u32)arg;
  2890. if (user_tsc_khz >= kvm_max_guest_tsc_khz)
  2891. goto out;
  2892. if (user_tsc_khz == 0)
  2893. user_tsc_khz = tsc_khz;
  2894. kvm_set_tsc_khz(vcpu, user_tsc_khz);
  2895. r = 0;
  2896. goto out;
  2897. }
  2898. case KVM_GET_TSC_KHZ: {
  2899. r = vcpu->arch.virtual_tsc_khz;
  2900. goto out;
  2901. }
  2902. case KVM_KVMCLOCK_CTRL: {
  2903. r = kvm_set_guest_paused(vcpu);
  2904. goto out;
  2905. }
  2906. default:
  2907. r = -EINVAL;
  2908. }
  2909. out:
  2910. kfree(u.buffer);
  2911. return r;
  2912. }
  2913. int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
  2914. {
  2915. return VM_FAULT_SIGBUS;
  2916. }
  2917. static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
  2918. {
  2919. int ret;
  2920. if (addr > (unsigned int)(-3 * PAGE_SIZE))
  2921. return -EINVAL;
  2922. ret = kvm_x86_ops->set_tss_addr(kvm, addr);
  2923. return ret;
  2924. }
  2925. static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
  2926. u64 ident_addr)
  2927. {
  2928. kvm->arch.ept_identity_map_addr = ident_addr;
  2929. return 0;
  2930. }
  2931. static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
  2932. u32 kvm_nr_mmu_pages)
  2933. {
  2934. if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
  2935. return -EINVAL;
  2936. mutex_lock(&kvm->slots_lock);
  2937. kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
  2938. kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
  2939. mutex_unlock(&kvm->slots_lock);
  2940. return 0;
  2941. }
  2942. static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
  2943. {
  2944. return kvm->arch.n_max_mmu_pages;
  2945. }
  2946. static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
  2947. {
  2948. int r;
  2949. r = 0;
  2950. switch (chip->chip_id) {
  2951. case KVM_IRQCHIP_PIC_MASTER:
  2952. memcpy(&chip->chip.pic,
  2953. &pic_irqchip(kvm)->pics[0],
  2954. sizeof(struct kvm_pic_state));
  2955. break;
  2956. case KVM_IRQCHIP_PIC_SLAVE:
  2957. memcpy(&chip->chip.pic,
  2958. &pic_irqchip(kvm)->pics[1],
  2959. sizeof(struct kvm_pic_state));
  2960. break;
  2961. case KVM_IRQCHIP_IOAPIC:
  2962. r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
  2963. break;
  2964. default:
  2965. r = -EINVAL;
  2966. break;
  2967. }
  2968. return r;
  2969. }
  2970. static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
  2971. {
  2972. int r;
  2973. r = 0;
  2974. switch (chip->chip_id) {
  2975. case KVM_IRQCHIP_PIC_MASTER:
  2976. spin_lock(&pic_irqchip(kvm)->lock);
  2977. memcpy(&pic_irqchip(kvm)->pics[0],
  2978. &chip->chip.pic,
  2979. sizeof(struct kvm_pic_state));
  2980. spin_unlock(&pic_irqchip(kvm)->lock);
  2981. break;
  2982. case KVM_IRQCHIP_PIC_SLAVE:
  2983. spin_lock(&pic_irqchip(kvm)->lock);
  2984. memcpy(&pic_irqchip(kvm)->pics[1],
  2985. &chip->chip.pic,
  2986. sizeof(struct kvm_pic_state));
  2987. spin_unlock(&pic_irqchip(kvm)->lock);
  2988. break;
  2989. case KVM_IRQCHIP_IOAPIC:
  2990. r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
  2991. break;
  2992. default:
  2993. r = -EINVAL;
  2994. break;
  2995. }
  2996. kvm_pic_update_irq(pic_irqchip(kvm));
  2997. return r;
  2998. }
  2999. static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
  3000. {
  3001. int r = 0;
  3002. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  3003. memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
  3004. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  3005. return r;
  3006. }
  3007. static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
  3008. {
  3009. int r = 0;
  3010. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  3011. memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
  3012. kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
  3013. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  3014. return r;
  3015. }
  3016. static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
  3017. {
  3018. int r = 0;
  3019. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  3020. memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
  3021. sizeof(ps->channels));
  3022. ps->flags = kvm->arch.vpit->pit_state.flags;
  3023. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  3024. memset(&ps->reserved, 0, sizeof(ps->reserved));
  3025. return r;
  3026. }
  3027. static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
  3028. {
  3029. int r = 0, start = 0;
  3030. u32 prev_legacy, cur_legacy;
  3031. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  3032. prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
  3033. cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
  3034. if (!prev_legacy && cur_legacy)
  3035. start = 1;
  3036. memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
  3037. sizeof(kvm->arch.vpit->pit_state.channels));
  3038. kvm->arch.vpit->pit_state.flags = ps->flags;
  3039. kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
  3040. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  3041. return r;
  3042. }
  3043. static int kvm_vm_ioctl_reinject(struct kvm *kvm,
  3044. struct kvm_reinject_control *control)
  3045. {
  3046. if (!kvm->arch.vpit)
  3047. return -ENXIO;
  3048. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  3049. kvm->arch.vpit->pit_state.reinject = control->pit_reinject;
  3050. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  3051. return 0;
  3052. }
  3053. /**
  3054. * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
  3055. * @kvm: kvm instance
  3056. * @log: slot id and address to which we copy the log
  3057. *
  3058. * We need to keep it in mind that VCPU threads can write to the bitmap
  3059. * concurrently. So, to avoid losing data, we keep the following order for
  3060. * each bit:
  3061. *
  3062. * 1. Take a snapshot of the bit and clear it if needed.
  3063. * 2. Write protect the corresponding page.
  3064. * 3. Flush TLB's if needed.
  3065. * 4. Copy the snapshot to the userspace.
  3066. *
  3067. * Between 2 and 3, the guest may write to the page using the remaining TLB
  3068. * entry. This is not a problem because the page will be reported dirty at
  3069. * step 4 using the snapshot taken before and step 3 ensures that successive
  3070. * writes will be logged for the next call.
  3071. */
  3072. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
  3073. {
  3074. int r;
  3075. struct kvm_memory_slot *memslot;
  3076. unsigned long n, i;
  3077. unsigned long *dirty_bitmap;
  3078. unsigned long *dirty_bitmap_buffer;
  3079. bool is_dirty = false;
  3080. mutex_lock(&kvm->slots_lock);
  3081. r = -EINVAL;
  3082. if (log->slot >= KVM_USER_MEM_SLOTS)
  3083. goto out;
  3084. memslot = id_to_memslot(kvm->memslots, log->slot);
  3085. dirty_bitmap = memslot->dirty_bitmap;
  3086. r = -ENOENT;
  3087. if (!dirty_bitmap)
  3088. goto out;
  3089. n = kvm_dirty_bitmap_bytes(memslot);
  3090. dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
  3091. memset(dirty_bitmap_buffer, 0, n);
  3092. spin_lock(&kvm->mmu_lock);
  3093. for (i = 0; i < n / sizeof(long); i++) {
  3094. unsigned long mask;
  3095. gfn_t offset;
  3096. if (!dirty_bitmap[i])
  3097. continue;
  3098. is_dirty = true;
  3099. mask = xchg(&dirty_bitmap[i], 0);
  3100. dirty_bitmap_buffer[i] = mask;
  3101. offset = i * BITS_PER_LONG;
  3102. kvm_mmu_write_protect_pt_masked(kvm, memslot, offset, mask);
  3103. }
  3104. if (is_dirty)
  3105. kvm_flush_remote_tlbs(kvm);
  3106. spin_unlock(&kvm->mmu_lock);
  3107. r = -EFAULT;
  3108. if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
  3109. goto out;
  3110. r = 0;
  3111. out:
  3112. mutex_unlock(&kvm->slots_lock);
  3113. return r;
  3114. }
  3115. int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
  3116. bool line_status)
  3117. {
  3118. if (!irqchip_in_kernel(kvm))
  3119. return -ENXIO;
  3120. irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
  3121. irq_event->irq, irq_event->level,
  3122. line_status);
  3123. return 0;
  3124. }
  3125. long kvm_arch_vm_ioctl(struct file *filp,
  3126. unsigned int ioctl, unsigned long arg)
  3127. {
  3128. struct kvm *kvm = filp->private_data;
  3129. void __user *argp = (void __user *)arg;
  3130. int r = -ENOTTY;
  3131. /*
  3132. * This union makes it completely explicit to gcc-3.x
  3133. * that these two variables' stack usage should be
  3134. * combined, not added together.
  3135. */
  3136. union {
  3137. struct kvm_pit_state ps;
  3138. struct kvm_pit_state2 ps2;
  3139. struct kvm_pit_config pit_config;
  3140. } u;
  3141. switch (ioctl) {
  3142. case KVM_SET_TSS_ADDR:
  3143. r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
  3144. break;
  3145. case KVM_SET_IDENTITY_MAP_ADDR: {
  3146. u64 ident_addr;
  3147. r = -EFAULT;
  3148. if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
  3149. goto out;
  3150. r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
  3151. break;
  3152. }
  3153. case KVM_SET_NR_MMU_PAGES:
  3154. r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
  3155. break;
  3156. case KVM_GET_NR_MMU_PAGES:
  3157. r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
  3158. break;
  3159. case KVM_CREATE_IRQCHIP: {
  3160. struct kvm_pic *vpic;
  3161. mutex_lock(&kvm->lock);
  3162. r = -EEXIST;
  3163. if (kvm->arch.vpic)
  3164. goto create_irqchip_unlock;
  3165. r = -EINVAL;
  3166. if (atomic_read(&kvm->online_vcpus))
  3167. goto create_irqchip_unlock;
  3168. r = -ENOMEM;
  3169. vpic = kvm_create_pic(kvm);
  3170. if (vpic) {
  3171. r = kvm_ioapic_init(kvm);
  3172. if (r) {
  3173. mutex_lock(&kvm->slots_lock);
  3174. kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
  3175. &vpic->dev_master);
  3176. kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
  3177. &vpic->dev_slave);
  3178. kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
  3179. &vpic->dev_eclr);
  3180. mutex_unlock(&kvm->slots_lock);
  3181. kfree(vpic);
  3182. goto create_irqchip_unlock;
  3183. }
  3184. } else
  3185. goto create_irqchip_unlock;
  3186. smp_wmb();
  3187. kvm->arch.vpic = vpic;
  3188. smp_wmb();
  3189. r = kvm_setup_default_irq_routing(kvm);
  3190. if (r) {
  3191. mutex_lock(&kvm->slots_lock);
  3192. mutex_lock(&kvm->irq_lock);
  3193. kvm_ioapic_destroy(kvm);
  3194. kvm_destroy_pic(kvm);
  3195. mutex_unlock(&kvm->irq_lock);
  3196. mutex_unlock(&kvm->slots_lock);
  3197. }
  3198. create_irqchip_unlock:
  3199. mutex_unlock(&kvm->lock);
  3200. break;
  3201. }
  3202. case KVM_CREATE_PIT:
  3203. u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
  3204. goto create_pit;
  3205. case KVM_CREATE_PIT2:
  3206. r = -EFAULT;
  3207. if (copy_from_user(&u.pit_config, argp,
  3208. sizeof(struct kvm_pit_config)))
  3209. goto out;
  3210. create_pit:
  3211. mutex_lock(&kvm->slots_lock);
  3212. r = -EEXIST;
  3213. if (kvm->arch.vpit)
  3214. goto create_pit_unlock;
  3215. r = -ENOMEM;
  3216. kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
  3217. if (kvm->arch.vpit)
  3218. r = 0;
  3219. create_pit_unlock:
  3220. mutex_unlock(&kvm->slots_lock);
  3221. break;
  3222. case KVM_GET_IRQCHIP: {
  3223. /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
  3224. struct kvm_irqchip *chip;
  3225. chip = memdup_user(argp, sizeof(*chip));
  3226. if (IS_ERR(chip)) {
  3227. r = PTR_ERR(chip);
  3228. goto out;
  3229. }
  3230. r = -ENXIO;
  3231. if (!irqchip_in_kernel(kvm))
  3232. goto get_irqchip_out;
  3233. r = kvm_vm_ioctl_get_irqchip(kvm, chip);
  3234. if (r)
  3235. goto get_irqchip_out;
  3236. r = -EFAULT;
  3237. if (copy_to_user(argp, chip, sizeof *chip))
  3238. goto get_irqchip_out;
  3239. r = 0;
  3240. get_irqchip_out:
  3241. kfree(chip);
  3242. break;
  3243. }
  3244. case KVM_SET_IRQCHIP: {
  3245. /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
  3246. struct kvm_irqchip *chip;
  3247. chip = memdup_user(argp, sizeof(*chip));
  3248. if (IS_ERR(chip)) {
  3249. r = PTR_ERR(chip);
  3250. goto out;
  3251. }
  3252. r = -ENXIO;
  3253. if (!irqchip_in_kernel(kvm))
  3254. goto set_irqchip_out;
  3255. r = kvm_vm_ioctl_set_irqchip(kvm, chip);
  3256. if (r)
  3257. goto set_irqchip_out;
  3258. r = 0;
  3259. set_irqchip_out:
  3260. kfree(chip);
  3261. break;
  3262. }
  3263. case KVM_GET_PIT: {
  3264. r = -EFAULT;
  3265. if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
  3266. goto out;
  3267. r = -ENXIO;
  3268. if (!kvm->arch.vpit)
  3269. goto out;
  3270. r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
  3271. if (r)
  3272. goto out;
  3273. r = -EFAULT;
  3274. if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
  3275. goto out;
  3276. r = 0;
  3277. break;
  3278. }
  3279. case KVM_SET_PIT: {
  3280. r = -EFAULT;
  3281. if (copy_from_user(&u.ps, argp, sizeof u.ps))
  3282. goto out;
  3283. r = -ENXIO;
  3284. if (!kvm->arch.vpit)
  3285. goto out;
  3286. r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
  3287. break;
  3288. }
  3289. case KVM_GET_PIT2: {
  3290. r = -ENXIO;
  3291. if (!kvm->arch.vpit)
  3292. goto out;
  3293. r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
  3294. if (r)
  3295. goto out;
  3296. r = -EFAULT;
  3297. if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
  3298. goto out;
  3299. r = 0;
  3300. break;
  3301. }
  3302. case KVM_SET_PIT2: {
  3303. r = -EFAULT;
  3304. if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
  3305. goto out;
  3306. r = -ENXIO;
  3307. if (!kvm->arch.vpit)
  3308. goto out;
  3309. r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
  3310. break;
  3311. }
  3312. case KVM_REINJECT_CONTROL: {
  3313. struct kvm_reinject_control control;
  3314. r = -EFAULT;
  3315. if (copy_from_user(&control, argp, sizeof(control)))
  3316. goto out;
  3317. r = kvm_vm_ioctl_reinject(kvm, &control);
  3318. break;
  3319. }
  3320. case KVM_XEN_HVM_CONFIG: {
  3321. r = -EFAULT;
  3322. if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
  3323. sizeof(struct kvm_xen_hvm_config)))
  3324. goto out;
  3325. r = -EINVAL;
  3326. if (kvm->arch.xen_hvm_config.flags)
  3327. goto out;
  3328. r = 0;
  3329. break;
  3330. }
  3331. case KVM_SET_CLOCK: {
  3332. struct kvm_clock_data user_ns;
  3333. u64 now_ns;
  3334. s64 delta;
  3335. r = -EFAULT;
  3336. if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
  3337. goto out;
  3338. r = -EINVAL;
  3339. if (user_ns.flags)
  3340. goto out;
  3341. r = 0;
  3342. local_irq_disable();
  3343. now_ns = get_kernel_ns();
  3344. delta = user_ns.clock - now_ns;
  3345. local_irq_enable();
  3346. kvm->arch.kvmclock_offset = delta;
  3347. break;
  3348. }
  3349. case KVM_GET_CLOCK: {
  3350. struct kvm_clock_data user_ns;
  3351. u64 now_ns;
  3352. local_irq_disable();
  3353. now_ns = get_kernel_ns();
  3354. user_ns.clock = kvm->arch.kvmclock_offset + now_ns;
  3355. local_irq_enable();
  3356. user_ns.flags = 0;
  3357. memset(&user_ns.pad, 0, sizeof(user_ns.pad));
  3358. r = -EFAULT;
  3359. if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
  3360. goto out;
  3361. r = 0;
  3362. break;
  3363. }
  3364. default:
  3365. ;
  3366. }
  3367. out:
  3368. return r;
  3369. }
  3370. static void kvm_init_msr_list(void)
  3371. {
  3372. u32 dummy[2];
  3373. unsigned i, j;
  3374. /* skip the first msrs in the list. KVM-specific */
  3375. for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) {
  3376. if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
  3377. continue;
  3378. if (j < i)
  3379. msrs_to_save[j] = msrs_to_save[i];
  3380. j++;
  3381. }
  3382. num_msrs_to_save = j;
  3383. }
  3384. static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
  3385. const void *v)
  3386. {
  3387. int handled = 0;
  3388. int n;
  3389. do {
  3390. n = min(len, 8);
  3391. if (!(vcpu->arch.apic &&
  3392. !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, n, v))
  3393. && kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
  3394. break;
  3395. handled += n;
  3396. addr += n;
  3397. len -= n;
  3398. v += n;
  3399. } while (len);
  3400. return handled;
  3401. }
  3402. static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
  3403. {
  3404. int handled = 0;
  3405. int n;
  3406. do {
  3407. n = min(len, 8);
  3408. if (!(vcpu->arch.apic &&
  3409. !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, n, v))
  3410. && kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
  3411. break;
  3412. trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v);
  3413. handled += n;
  3414. addr += n;
  3415. len -= n;
  3416. v += n;
  3417. } while (len);
  3418. return handled;
  3419. }
  3420. static void kvm_set_segment(struct kvm_vcpu *vcpu,
  3421. struct kvm_segment *var, int seg)
  3422. {
  3423. kvm_x86_ops->set_segment(vcpu, var, seg);
  3424. }
  3425. void kvm_get_segment(struct kvm_vcpu *vcpu,
  3426. struct kvm_segment *var, int seg)
  3427. {
  3428. kvm_x86_ops->get_segment(vcpu, var, seg);
  3429. }
  3430. gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
  3431. {
  3432. gpa_t t_gpa;
  3433. struct x86_exception exception;
  3434. BUG_ON(!mmu_is_nested(vcpu));
  3435. /* NPT walks are always user-walks */
  3436. access |= PFERR_USER_MASK;
  3437. t_gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &exception);
  3438. return t_gpa;
  3439. }
  3440. gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
  3441. struct x86_exception *exception)
  3442. {
  3443. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3444. return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
  3445. }
  3446. gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
  3447. struct x86_exception *exception)
  3448. {
  3449. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3450. access |= PFERR_FETCH_MASK;
  3451. return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
  3452. }
  3453. gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
  3454. struct x86_exception *exception)
  3455. {
  3456. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3457. access |= PFERR_WRITE_MASK;
  3458. return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
  3459. }
  3460. /* uses this to access any guest's mapped memory without checking CPL */
  3461. gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
  3462. struct x86_exception *exception)
  3463. {
  3464. return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
  3465. }
  3466. static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
  3467. struct kvm_vcpu *vcpu, u32 access,
  3468. struct x86_exception *exception)
  3469. {
  3470. void *data = val;
  3471. int r = X86EMUL_CONTINUE;
  3472. while (bytes) {
  3473. gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
  3474. exception);
  3475. unsigned offset = addr & (PAGE_SIZE-1);
  3476. unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
  3477. int ret;
  3478. if (gpa == UNMAPPED_GVA)
  3479. return X86EMUL_PROPAGATE_FAULT;
  3480. ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
  3481. if (ret < 0) {
  3482. r = X86EMUL_IO_NEEDED;
  3483. goto out;
  3484. }
  3485. bytes -= toread;
  3486. data += toread;
  3487. addr += toread;
  3488. }
  3489. out:
  3490. return r;
  3491. }
  3492. /* used for instruction fetching */
  3493. static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
  3494. gva_t addr, void *val, unsigned int bytes,
  3495. struct x86_exception *exception)
  3496. {
  3497. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3498. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3499. return kvm_read_guest_virt_helper(addr, val, bytes, vcpu,
  3500. access | PFERR_FETCH_MASK,
  3501. exception);
  3502. }
  3503. int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
  3504. gva_t addr, void *val, unsigned int bytes,
  3505. struct x86_exception *exception)
  3506. {
  3507. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3508. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  3509. return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
  3510. exception);
  3511. }
  3512. EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
  3513. static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt,
  3514. gva_t addr, void *val, unsigned int bytes,
  3515. struct x86_exception *exception)
  3516. {
  3517. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3518. return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception);
  3519. }
  3520. int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
  3521. gva_t addr, void *val,
  3522. unsigned int bytes,
  3523. struct x86_exception *exception)
  3524. {
  3525. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3526. void *data = val;
  3527. int r = X86EMUL_CONTINUE;
  3528. while (bytes) {
  3529. gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
  3530. PFERR_WRITE_MASK,
  3531. exception);
  3532. unsigned offset = addr & (PAGE_SIZE-1);
  3533. unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
  3534. int ret;
  3535. if (gpa == UNMAPPED_GVA)
  3536. return X86EMUL_PROPAGATE_FAULT;
  3537. ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
  3538. if (ret < 0) {
  3539. r = X86EMUL_IO_NEEDED;
  3540. goto out;
  3541. }
  3542. bytes -= towrite;
  3543. data += towrite;
  3544. addr += towrite;
  3545. }
  3546. out:
  3547. return r;
  3548. }
  3549. EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
  3550. static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
  3551. gpa_t *gpa, struct x86_exception *exception,
  3552. bool write)
  3553. {
  3554. u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
  3555. | (write ? PFERR_WRITE_MASK : 0);
  3556. if (vcpu_match_mmio_gva(vcpu, gva)
  3557. && !permission_fault(vcpu->arch.walk_mmu, vcpu->arch.access, access)) {
  3558. *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
  3559. (gva & (PAGE_SIZE - 1));
  3560. trace_vcpu_match_mmio(gva, *gpa, write, false);
  3561. return 1;
  3562. }
  3563. *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
  3564. if (*gpa == UNMAPPED_GVA)
  3565. return -1;
  3566. /* For APIC access vmexit */
  3567. if ((*gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  3568. return 1;
  3569. if (vcpu_match_mmio_gpa(vcpu, *gpa)) {
  3570. trace_vcpu_match_mmio(gva, *gpa, write, true);
  3571. return 1;
  3572. }
  3573. return 0;
  3574. }
  3575. int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
  3576. const void *val, int bytes)
  3577. {
  3578. int ret;
  3579. ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
  3580. if (ret < 0)
  3581. return 0;
  3582. kvm_mmu_pte_write(vcpu, gpa, val, bytes);
  3583. return 1;
  3584. }
  3585. struct read_write_emulator_ops {
  3586. int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
  3587. int bytes);
  3588. int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
  3589. void *val, int bytes);
  3590. int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
  3591. int bytes, void *val);
  3592. int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
  3593. void *val, int bytes);
  3594. bool write;
  3595. };
  3596. static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
  3597. {
  3598. if (vcpu->mmio_read_completed) {
  3599. trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
  3600. vcpu->mmio_fragments[0].gpa, *(u64 *)val);
  3601. vcpu->mmio_read_completed = 0;
  3602. return 1;
  3603. }
  3604. return 0;
  3605. }
  3606. static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
  3607. void *val, int bytes)
  3608. {
  3609. return !kvm_read_guest(vcpu->kvm, gpa, val, bytes);
  3610. }
  3611. static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
  3612. void *val, int bytes)
  3613. {
  3614. return emulator_write_phys(vcpu, gpa, val, bytes);
  3615. }
  3616. static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
  3617. {
  3618. trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
  3619. return vcpu_mmio_write(vcpu, gpa, bytes, val);
  3620. }
  3621. static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
  3622. void *val, int bytes)
  3623. {
  3624. trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
  3625. return X86EMUL_IO_NEEDED;
  3626. }
  3627. static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
  3628. void *val, int bytes)
  3629. {
  3630. struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
  3631. memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
  3632. return X86EMUL_CONTINUE;
  3633. }
  3634. static const struct read_write_emulator_ops read_emultor = {
  3635. .read_write_prepare = read_prepare,
  3636. .read_write_emulate = read_emulate,
  3637. .read_write_mmio = vcpu_mmio_read,
  3638. .read_write_exit_mmio = read_exit_mmio,
  3639. };
  3640. static const struct read_write_emulator_ops write_emultor = {
  3641. .read_write_emulate = write_emulate,
  3642. .read_write_mmio = write_mmio,
  3643. .read_write_exit_mmio = write_exit_mmio,
  3644. .write = true,
  3645. };
  3646. static int emulator_read_write_onepage(unsigned long addr, void *val,
  3647. unsigned int bytes,
  3648. struct x86_exception *exception,
  3649. struct kvm_vcpu *vcpu,
  3650. const struct read_write_emulator_ops *ops)
  3651. {
  3652. gpa_t gpa;
  3653. int handled, ret;
  3654. bool write = ops->write;
  3655. struct kvm_mmio_fragment *frag;
  3656. ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
  3657. if (ret < 0)
  3658. return X86EMUL_PROPAGATE_FAULT;
  3659. /* For APIC access vmexit */
  3660. if (ret)
  3661. goto mmio;
  3662. if (ops->read_write_emulate(vcpu, gpa, val, bytes))
  3663. return X86EMUL_CONTINUE;
  3664. mmio:
  3665. /*
  3666. * Is this MMIO handled locally?
  3667. */
  3668. handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
  3669. if (handled == bytes)
  3670. return X86EMUL_CONTINUE;
  3671. gpa += handled;
  3672. bytes -= handled;
  3673. val += handled;
  3674. WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
  3675. frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
  3676. frag->gpa = gpa;
  3677. frag->data = val;
  3678. frag->len = bytes;
  3679. return X86EMUL_CONTINUE;
  3680. }
  3681. int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr,
  3682. void *val, unsigned int bytes,
  3683. struct x86_exception *exception,
  3684. const struct read_write_emulator_ops *ops)
  3685. {
  3686. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3687. gpa_t gpa;
  3688. int rc;
  3689. if (ops->read_write_prepare &&
  3690. ops->read_write_prepare(vcpu, val, bytes))
  3691. return X86EMUL_CONTINUE;
  3692. vcpu->mmio_nr_fragments = 0;
  3693. /* Crossing a page boundary? */
  3694. if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
  3695. int now;
  3696. now = -addr & ~PAGE_MASK;
  3697. rc = emulator_read_write_onepage(addr, val, now, exception,
  3698. vcpu, ops);
  3699. if (rc != X86EMUL_CONTINUE)
  3700. return rc;
  3701. addr += now;
  3702. val += now;
  3703. bytes -= now;
  3704. }
  3705. rc = emulator_read_write_onepage(addr, val, bytes, exception,
  3706. vcpu, ops);
  3707. if (rc != X86EMUL_CONTINUE)
  3708. return rc;
  3709. if (!vcpu->mmio_nr_fragments)
  3710. return rc;
  3711. gpa = vcpu->mmio_fragments[0].gpa;
  3712. vcpu->mmio_needed = 1;
  3713. vcpu->mmio_cur_fragment = 0;
  3714. vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
  3715. vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
  3716. vcpu->run->exit_reason = KVM_EXIT_MMIO;
  3717. vcpu->run->mmio.phys_addr = gpa;
  3718. return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
  3719. }
  3720. static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
  3721. unsigned long addr,
  3722. void *val,
  3723. unsigned int bytes,
  3724. struct x86_exception *exception)
  3725. {
  3726. return emulator_read_write(ctxt, addr, val, bytes,
  3727. exception, &read_emultor);
  3728. }
  3729. int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
  3730. unsigned long addr,
  3731. const void *val,
  3732. unsigned int bytes,
  3733. struct x86_exception *exception)
  3734. {
  3735. return emulator_read_write(ctxt, addr, (void *)val, bytes,
  3736. exception, &write_emultor);
  3737. }
  3738. #define CMPXCHG_TYPE(t, ptr, old, new) \
  3739. (cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))
  3740. #ifdef CONFIG_X86_64
  3741. # define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
  3742. #else
  3743. # define CMPXCHG64(ptr, old, new) \
  3744. (cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
  3745. #endif
  3746. static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
  3747. unsigned long addr,
  3748. const void *old,
  3749. const void *new,
  3750. unsigned int bytes,
  3751. struct x86_exception *exception)
  3752. {
  3753. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3754. gpa_t gpa;
  3755. struct page *page;
  3756. char *kaddr;
  3757. bool exchanged;
  3758. /* guests cmpxchg8b have to be emulated atomically */
  3759. if (bytes > 8 || (bytes & (bytes - 1)))
  3760. goto emul_write;
  3761. gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
  3762. if (gpa == UNMAPPED_GVA ||
  3763. (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  3764. goto emul_write;
  3765. if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
  3766. goto emul_write;
  3767. page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  3768. if (is_error_page(page))
  3769. goto emul_write;
  3770. kaddr = kmap_atomic(page);
  3771. kaddr += offset_in_page(gpa);
  3772. switch (bytes) {
  3773. case 1:
  3774. exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
  3775. break;
  3776. case 2:
  3777. exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
  3778. break;
  3779. case 4:
  3780. exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
  3781. break;
  3782. case 8:
  3783. exchanged = CMPXCHG64(kaddr, old, new);
  3784. break;
  3785. default:
  3786. BUG();
  3787. }
  3788. kunmap_atomic(kaddr);
  3789. kvm_release_page_dirty(page);
  3790. if (!exchanged)
  3791. return X86EMUL_CMPXCHG_FAILED;
  3792. kvm_mmu_pte_write(vcpu, gpa, new, bytes);
  3793. return X86EMUL_CONTINUE;
  3794. emul_write:
  3795. printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
  3796. return emulator_write_emulated(ctxt, addr, new, bytes, exception);
  3797. }
  3798. static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
  3799. {
  3800. /* TODO: String I/O for in kernel device */
  3801. int r;
  3802. if (vcpu->arch.pio.in)
  3803. r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
  3804. vcpu->arch.pio.size, pd);
  3805. else
  3806. r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
  3807. vcpu->arch.pio.port, vcpu->arch.pio.size,
  3808. pd);
  3809. return r;
  3810. }
  3811. static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
  3812. unsigned short port, void *val,
  3813. unsigned int count, bool in)
  3814. {
  3815. trace_kvm_pio(!in, port, size, count);
  3816. vcpu->arch.pio.port = port;
  3817. vcpu->arch.pio.in = in;
  3818. vcpu->arch.pio.count = count;
  3819. vcpu->arch.pio.size = size;
  3820. if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
  3821. vcpu->arch.pio.count = 0;
  3822. return 1;
  3823. }
  3824. vcpu->run->exit_reason = KVM_EXIT_IO;
  3825. vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  3826. vcpu->run->io.size = size;
  3827. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  3828. vcpu->run->io.count = count;
  3829. vcpu->run->io.port = port;
  3830. return 0;
  3831. }
  3832. static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
  3833. int size, unsigned short port, void *val,
  3834. unsigned int count)
  3835. {
  3836. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3837. int ret;
  3838. if (vcpu->arch.pio.count)
  3839. goto data_avail;
  3840. ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
  3841. if (ret) {
  3842. data_avail:
  3843. memcpy(val, vcpu->arch.pio_data, size * count);
  3844. vcpu->arch.pio.count = 0;
  3845. return 1;
  3846. }
  3847. return 0;
  3848. }
  3849. static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
  3850. int size, unsigned short port,
  3851. const void *val, unsigned int count)
  3852. {
  3853. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3854. memcpy(vcpu->arch.pio_data, val, size * count);
  3855. return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
  3856. }
  3857. static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
  3858. {
  3859. return kvm_x86_ops->get_segment_base(vcpu, seg);
  3860. }
  3861. static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
  3862. {
  3863. kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
  3864. }
  3865. int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
  3866. {
  3867. if (!need_emulate_wbinvd(vcpu))
  3868. return X86EMUL_CONTINUE;
  3869. if (kvm_x86_ops->has_wbinvd_exit()) {
  3870. int cpu = get_cpu();
  3871. cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
  3872. smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
  3873. wbinvd_ipi, NULL, 1);
  3874. put_cpu();
  3875. cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
  3876. } else
  3877. wbinvd();
  3878. return X86EMUL_CONTINUE;
  3879. }
  3880. EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
  3881. static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
  3882. {
  3883. kvm_emulate_wbinvd(emul_to_vcpu(ctxt));
  3884. }
  3885. int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
  3886. {
  3887. return _kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
  3888. }
  3889. int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
  3890. {
  3891. return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
  3892. }
  3893. static u64 mk_cr_64(u64 curr_cr, u32 new_val)
  3894. {
  3895. return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
  3896. }
  3897. static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
  3898. {
  3899. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3900. unsigned long value;
  3901. switch (cr) {
  3902. case 0:
  3903. value = kvm_read_cr0(vcpu);
  3904. break;
  3905. case 2:
  3906. value = vcpu->arch.cr2;
  3907. break;
  3908. case 3:
  3909. value = kvm_read_cr3(vcpu);
  3910. break;
  3911. case 4:
  3912. value = kvm_read_cr4(vcpu);
  3913. break;
  3914. case 8:
  3915. value = kvm_get_cr8(vcpu);
  3916. break;
  3917. default:
  3918. kvm_err("%s: unexpected cr %u\n", __func__, cr);
  3919. return 0;
  3920. }
  3921. return value;
  3922. }
  3923. static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
  3924. {
  3925. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  3926. int res = 0;
  3927. switch (cr) {
  3928. case 0:
  3929. res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
  3930. break;
  3931. case 2:
  3932. vcpu->arch.cr2 = val;
  3933. break;
  3934. case 3:
  3935. res = kvm_set_cr3(vcpu, val);
  3936. break;
  3937. case 4:
  3938. res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
  3939. break;
  3940. case 8:
  3941. res = kvm_set_cr8(vcpu, val);
  3942. break;
  3943. default:
  3944. kvm_err("%s: unexpected cr %u\n", __func__, cr);
  3945. res = -1;
  3946. }
  3947. return res;
  3948. }
  3949. static void emulator_set_rflags(struct x86_emulate_ctxt *ctxt, ulong val)
  3950. {
  3951. kvm_set_rflags(emul_to_vcpu(ctxt), val);
  3952. }
  3953. static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
  3954. {
  3955. return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
  3956. }
  3957. static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
  3958. {
  3959. kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
  3960. }
  3961. static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
  3962. {
  3963. kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
  3964. }
  3965. static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
  3966. {
  3967. kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
  3968. }
  3969. static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
  3970. {
  3971. kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
  3972. }
  3973. static unsigned long emulator_get_cached_segment_base(
  3974. struct x86_emulate_ctxt *ctxt, int seg)
  3975. {
  3976. return get_segment_base(emul_to_vcpu(ctxt), seg);
  3977. }
  3978. static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
  3979. struct desc_struct *desc, u32 *base3,
  3980. int seg)
  3981. {
  3982. struct kvm_segment var;
  3983. kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
  3984. *selector = var.selector;
  3985. if (var.unusable) {
  3986. memset(desc, 0, sizeof(*desc));
  3987. return false;
  3988. }
  3989. if (var.g)
  3990. var.limit >>= 12;
  3991. set_desc_limit(desc, var.limit);
  3992. set_desc_base(desc, (unsigned long)var.base);
  3993. #ifdef CONFIG_X86_64
  3994. if (base3)
  3995. *base3 = var.base >> 32;
  3996. #endif
  3997. desc->type = var.type;
  3998. desc->s = var.s;
  3999. desc->dpl = var.dpl;
  4000. desc->p = var.present;
  4001. desc->avl = var.avl;
  4002. desc->l = var.l;
  4003. desc->d = var.db;
  4004. desc->g = var.g;
  4005. return true;
  4006. }
  4007. static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
  4008. struct desc_struct *desc, u32 base3,
  4009. int seg)
  4010. {
  4011. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  4012. struct kvm_segment var;
  4013. var.selector = selector;
  4014. var.base = get_desc_base(desc);
  4015. #ifdef CONFIG_X86_64
  4016. var.base |= ((u64)base3) << 32;
  4017. #endif
  4018. var.limit = get_desc_limit(desc);
  4019. if (desc->g)
  4020. var.limit = (var.limit << 12) | 0xfff;
  4021. var.type = desc->type;
  4022. var.present = desc->p;
  4023. var.dpl = desc->dpl;
  4024. var.db = desc->d;
  4025. var.s = desc->s;
  4026. var.l = desc->l;
  4027. var.g = desc->g;
  4028. var.avl = desc->avl;
  4029. var.present = desc->p;
  4030. var.unusable = !var.present;
  4031. var.padding = 0;
  4032. kvm_set_segment(vcpu, &var, seg);
  4033. return;
  4034. }
  4035. static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
  4036. u32 msr_index, u64 *pdata)
  4037. {
  4038. return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
  4039. }
  4040. static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
  4041. u32 msr_index, u64 data)
  4042. {
  4043. struct msr_data msr;
  4044. msr.data = data;
  4045. msr.index = msr_index;
  4046. msr.host_initiated = false;
  4047. return kvm_set_msr(emul_to_vcpu(ctxt), &msr);
  4048. }
  4049. static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
  4050. u32 pmc, u64 *pdata)
  4051. {
  4052. return kvm_pmu_read_pmc(emul_to_vcpu(ctxt), pmc, pdata);
  4053. }
  4054. static void emulator_halt(struct x86_emulate_ctxt *ctxt)
  4055. {
  4056. emul_to_vcpu(ctxt)->arch.halt_request = 1;
  4057. }
  4058. static void emulator_get_fpu(struct x86_emulate_ctxt *ctxt)
  4059. {
  4060. preempt_disable();
  4061. kvm_load_guest_fpu(emul_to_vcpu(ctxt));
  4062. /*
  4063. * CR0.TS may reference the host fpu state, not the guest fpu state,
  4064. * so it may be clear at this point.
  4065. */
  4066. clts();
  4067. }
  4068. static void emulator_put_fpu(struct x86_emulate_ctxt *ctxt)
  4069. {
  4070. preempt_enable();
  4071. }
  4072. static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
  4073. struct x86_instruction_info *info,
  4074. enum x86_intercept_stage stage)
  4075. {
  4076. return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
  4077. }
  4078. static void emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
  4079. u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
  4080. {
  4081. kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx);
  4082. }
  4083. static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
  4084. {
  4085. return kvm_register_read(emul_to_vcpu(ctxt), reg);
  4086. }
  4087. static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
  4088. {
  4089. kvm_register_write(emul_to_vcpu(ctxt), reg, val);
  4090. }
  4091. static const struct x86_emulate_ops emulate_ops = {
  4092. .read_gpr = emulator_read_gpr,
  4093. .write_gpr = emulator_write_gpr,
  4094. .read_std = kvm_read_guest_virt_system,
  4095. .write_std = kvm_write_guest_virt_system,
  4096. .fetch = kvm_fetch_guest_virt,
  4097. .read_emulated = emulator_read_emulated,
  4098. .write_emulated = emulator_write_emulated,
  4099. .cmpxchg_emulated = emulator_cmpxchg_emulated,
  4100. .invlpg = emulator_invlpg,
  4101. .pio_in_emulated = emulator_pio_in_emulated,
  4102. .pio_out_emulated = emulator_pio_out_emulated,
  4103. .get_segment = emulator_get_segment,
  4104. .set_segment = emulator_set_segment,
  4105. .get_cached_segment_base = emulator_get_cached_segment_base,
  4106. .get_gdt = emulator_get_gdt,
  4107. .get_idt = emulator_get_idt,
  4108. .set_gdt = emulator_set_gdt,
  4109. .set_idt = emulator_set_idt,
  4110. .get_cr = emulator_get_cr,
  4111. .set_cr = emulator_set_cr,
  4112. .set_rflags = emulator_set_rflags,
  4113. .cpl = emulator_get_cpl,
  4114. .get_dr = emulator_get_dr,
  4115. .set_dr = emulator_set_dr,
  4116. .set_msr = emulator_set_msr,
  4117. .get_msr = emulator_get_msr,
  4118. .read_pmc = emulator_read_pmc,
  4119. .halt = emulator_halt,
  4120. .wbinvd = emulator_wbinvd,
  4121. .fix_hypercall = emulator_fix_hypercall,
  4122. .get_fpu = emulator_get_fpu,
  4123. .put_fpu = emulator_put_fpu,
  4124. .intercept = emulator_intercept,
  4125. .get_cpuid = emulator_get_cpuid,
  4126. };
  4127. static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
  4128. {
  4129. u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask);
  4130. /*
  4131. * an sti; sti; sequence only disable interrupts for the first
  4132. * instruction. So, if the last instruction, be it emulated or
  4133. * not, left the system with the INT_STI flag enabled, it
  4134. * means that the last instruction is an sti. We should not
  4135. * leave the flag on in this case. The same goes for mov ss
  4136. */
  4137. if (!(int_shadow & mask))
  4138. kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
  4139. }
  4140. static void inject_emulated_exception(struct kvm_vcpu *vcpu)
  4141. {
  4142. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  4143. if (ctxt->exception.vector == PF_VECTOR)
  4144. kvm_propagate_fault(vcpu, &ctxt->exception);
  4145. else if (ctxt->exception.error_code_valid)
  4146. kvm_queue_exception_e(vcpu, ctxt->exception.vector,
  4147. ctxt->exception.error_code);
  4148. else
  4149. kvm_queue_exception(vcpu, ctxt->exception.vector);
  4150. }
  4151. static void init_decode_cache(struct x86_emulate_ctxt *ctxt)
  4152. {
  4153. memset(&ctxt->twobyte, 0,
  4154. (void *)&ctxt->_regs - (void *)&ctxt->twobyte);
  4155. ctxt->fetch.start = 0;
  4156. ctxt->fetch.end = 0;
  4157. ctxt->io_read.pos = 0;
  4158. ctxt->io_read.end = 0;
  4159. ctxt->mem_read.pos = 0;
  4160. ctxt->mem_read.end = 0;
  4161. }
  4162. static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
  4163. {
  4164. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  4165. int cs_db, cs_l;
  4166. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  4167. ctxt->eflags = kvm_get_rflags(vcpu);
  4168. ctxt->eip = kvm_rip_read(vcpu);
  4169. ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL :
  4170. (ctxt->eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 :
  4171. cs_l ? X86EMUL_MODE_PROT64 :
  4172. cs_db ? X86EMUL_MODE_PROT32 :
  4173. X86EMUL_MODE_PROT16;
  4174. ctxt->guest_mode = is_guest_mode(vcpu);
  4175. init_decode_cache(ctxt);
  4176. vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
  4177. }
  4178. int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
  4179. {
  4180. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  4181. int ret;
  4182. init_emulate_ctxt(vcpu);
  4183. ctxt->op_bytes = 2;
  4184. ctxt->ad_bytes = 2;
  4185. ctxt->_eip = ctxt->eip + inc_eip;
  4186. ret = emulate_int_real(ctxt, irq);
  4187. if (ret != X86EMUL_CONTINUE)
  4188. return EMULATE_FAIL;
  4189. ctxt->eip = ctxt->_eip;
  4190. kvm_rip_write(vcpu, ctxt->eip);
  4191. kvm_set_rflags(vcpu, ctxt->eflags);
  4192. if (irq == NMI_VECTOR)
  4193. vcpu->arch.nmi_pending = 0;
  4194. else
  4195. vcpu->arch.interrupt.pending = false;
  4196. return EMULATE_DONE;
  4197. }
  4198. EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);
  4199. static int handle_emulation_failure(struct kvm_vcpu *vcpu)
  4200. {
  4201. int r = EMULATE_DONE;
  4202. ++vcpu->stat.insn_emulation_fail;
  4203. trace_kvm_emulate_insn_failed(vcpu);
  4204. if (!is_guest_mode(vcpu)) {
  4205. vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  4206. vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
  4207. vcpu->run->internal.ndata = 0;
  4208. r = EMULATE_FAIL;
  4209. }
  4210. kvm_queue_exception(vcpu, UD_VECTOR);
  4211. return r;
  4212. }
  4213. static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,
  4214. bool write_fault_to_shadow_pgtable,
  4215. int emulation_type)
  4216. {
  4217. gpa_t gpa = cr2;
  4218. pfn_t pfn;
  4219. if (emulation_type & EMULTYPE_NO_REEXECUTE)
  4220. return false;
  4221. if (!vcpu->arch.mmu.direct_map) {
  4222. /*
  4223. * Write permission should be allowed since only
  4224. * write access need to be emulated.
  4225. */
  4226. gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
  4227. /*
  4228. * If the mapping is invalid in guest, let cpu retry
  4229. * it to generate fault.
  4230. */
  4231. if (gpa == UNMAPPED_GVA)
  4232. return true;
  4233. }
  4234. /*
  4235. * Do not retry the unhandleable instruction if it faults on the
  4236. * readonly host memory, otherwise it will goto a infinite loop:
  4237. * retry instruction -> write #PF -> emulation fail -> retry
  4238. * instruction -> ...
  4239. */
  4240. pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
  4241. /*
  4242. * If the instruction failed on the error pfn, it can not be fixed,
  4243. * report the error to userspace.
  4244. */
  4245. if (is_error_noslot_pfn(pfn))
  4246. return false;
  4247. kvm_release_pfn_clean(pfn);
  4248. /* The instructions are well-emulated on direct mmu. */
  4249. if (vcpu->arch.mmu.direct_map) {
  4250. unsigned int indirect_shadow_pages;
  4251. spin_lock(&vcpu->kvm->mmu_lock);
  4252. indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
  4253. spin_unlock(&vcpu->kvm->mmu_lock);
  4254. if (indirect_shadow_pages)
  4255. kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
  4256. return true;
  4257. }
  4258. /*
  4259. * if emulation was due to access to shadowed page table
  4260. * and it failed try to unshadow page and re-enter the
  4261. * guest to let CPU execute the instruction.
  4262. */
  4263. kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
  4264. /*
  4265. * If the access faults on its page table, it can not
  4266. * be fixed by unprotecting shadow page and it should
  4267. * be reported to userspace.
  4268. */
  4269. return !write_fault_to_shadow_pgtable;
  4270. }
  4271. static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
  4272. unsigned long cr2, int emulation_type)
  4273. {
  4274. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  4275. unsigned long last_retry_eip, last_retry_addr, gpa = cr2;
  4276. last_retry_eip = vcpu->arch.last_retry_eip;
  4277. last_retry_addr = vcpu->arch.last_retry_addr;
  4278. /*
  4279. * If the emulation is caused by #PF and it is non-page_table
  4280. * writing instruction, it means the VM-EXIT is caused by shadow
  4281. * page protected, we can zap the shadow page and retry this
  4282. * instruction directly.
  4283. *
  4284. * Note: if the guest uses a non-page-table modifying instruction
  4285. * on the PDE that points to the instruction, then we will unmap
  4286. * the instruction and go to an infinite loop. So, we cache the
  4287. * last retried eip and the last fault address, if we meet the eip
  4288. * and the address again, we can break out of the potential infinite
  4289. * loop.
  4290. */
  4291. vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
  4292. if (!(emulation_type & EMULTYPE_RETRY))
  4293. return false;
  4294. if (x86_page_table_writing_insn(ctxt))
  4295. return false;
  4296. if (ctxt->eip == last_retry_eip && last_retry_addr == cr2)
  4297. return false;
  4298. vcpu->arch.last_retry_eip = ctxt->eip;
  4299. vcpu->arch.last_retry_addr = cr2;
  4300. if (!vcpu->arch.mmu.direct_map)
  4301. gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
  4302. kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
  4303. return true;
  4304. }
  4305. static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
  4306. static int complete_emulated_pio(struct kvm_vcpu *vcpu);
  4307. int x86_emulate_instruction(struct kvm_vcpu *vcpu,
  4308. unsigned long cr2,
  4309. int emulation_type,
  4310. void *insn,
  4311. int insn_len)
  4312. {
  4313. int r;
  4314. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  4315. bool writeback = true;
  4316. bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
  4317. /*
  4318. * Clear write_fault_to_shadow_pgtable here to ensure it is
  4319. * never reused.
  4320. */
  4321. vcpu->arch.write_fault_to_shadow_pgtable = false;
  4322. kvm_clear_exception_queue(vcpu);
  4323. if (!(emulation_type & EMULTYPE_NO_DECODE)) {
  4324. init_emulate_ctxt(vcpu);
  4325. ctxt->interruptibility = 0;
  4326. ctxt->have_exception = false;
  4327. ctxt->perm_ok = false;
  4328. ctxt->only_vendor_specific_insn
  4329. = emulation_type & EMULTYPE_TRAP_UD;
  4330. r = x86_decode_insn(ctxt, insn, insn_len);
  4331. trace_kvm_emulate_insn_start(vcpu);
  4332. ++vcpu->stat.insn_emulation;
  4333. if (r != EMULATION_OK) {
  4334. if (emulation_type & EMULTYPE_TRAP_UD)
  4335. return EMULATE_FAIL;
  4336. if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
  4337. emulation_type))
  4338. return EMULATE_DONE;
  4339. if (emulation_type & EMULTYPE_SKIP)
  4340. return EMULATE_FAIL;
  4341. return handle_emulation_failure(vcpu);
  4342. }
  4343. }
  4344. if (emulation_type & EMULTYPE_SKIP) {
  4345. kvm_rip_write(vcpu, ctxt->_eip);
  4346. return EMULATE_DONE;
  4347. }
  4348. if (retry_instruction(ctxt, cr2, emulation_type))
  4349. return EMULATE_DONE;
  4350. /* this is needed for vmware backdoor interface to work since it
  4351. changes registers values during IO operation */
  4352. if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
  4353. vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
  4354. emulator_invalidate_register_cache(ctxt);
  4355. }
  4356. restart:
  4357. r = x86_emulate_insn(ctxt);
  4358. if (r == EMULATION_INTERCEPTED)
  4359. return EMULATE_DONE;
  4360. if (r == EMULATION_FAILED) {
  4361. if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
  4362. emulation_type))
  4363. return EMULATE_DONE;
  4364. return handle_emulation_failure(vcpu);
  4365. }
  4366. if (ctxt->have_exception) {
  4367. inject_emulated_exception(vcpu);
  4368. r = EMULATE_DONE;
  4369. } else if (vcpu->arch.pio.count) {
  4370. if (!vcpu->arch.pio.in)
  4371. vcpu->arch.pio.count = 0;
  4372. else {
  4373. writeback = false;
  4374. vcpu->arch.complete_userspace_io = complete_emulated_pio;
  4375. }
  4376. r = EMULATE_DO_MMIO;
  4377. } else if (vcpu->mmio_needed) {
  4378. if (!vcpu->mmio_is_write)
  4379. writeback = false;
  4380. r = EMULATE_DO_MMIO;
  4381. vcpu->arch.complete_userspace_io = complete_emulated_mmio;
  4382. } else if (r == EMULATION_RESTART)
  4383. goto restart;
  4384. else
  4385. r = EMULATE_DONE;
  4386. if (writeback) {
  4387. toggle_interruptibility(vcpu, ctxt->interruptibility);
  4388. kvm_set_rflags(vcpu, ctxt->eflags);
  4389. kvm_make_request(KVM_REQ_EVENT, vcpu);
  4390. vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
  4391. kvm_rip_write(vcpu, ctxt->eip);
  4392. } else
  4393. vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
  4394. return r;
  4395. }
  4396. EXPORT_SYMBOL_GPL(x86_emulate_instruction);
  4397. int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
  4398. {
  4399. unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX);
  4400. int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
  4401. size, port, &val, 1);
  4402. /* do not return to emulator after return from userspace */
  4403. vcpu->arch.pio.count = 0;
  4404. return ret;
  4405. }
  4406. EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
  4407. static void tsc_bad(void *info)
  4408. {
  4409. __this_cpu_write(cpu_tsc_khz, 0);
  4410. }
  4411. static void tsc_khz_changed(void *data)
  4412. {
  4413. struct cpufreq_freqs *freq = data;
  4414. unsigned long khz = 0;
  4415. if (data)
  4416. khz = freq->new;
  4417. else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
  4418. khz = cpufreq_quick_get(raw_smp_processor_id());
  4419. if (!khz)
  4420. khz = tsc_khz;
  4421. __this_cpu_write(cpu_tsc_khz, khz);
  4422. }
  4423. static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
  4424. void *data)
  4425. {
  4426. struct cpufreq_freqs *freq = data;
  4427. struct kvm *kvm;
  4428. struct kvm_vcpu *vcpu;
  4429. int i, send_ipi = 0;
  4430. /*
  4431. * We allow guests to temporarily run on slowing clocks,
  4432. * provided we notify them after, or to run on accelerating
  4433. * clocks, provided we notify them before. Thus time never
  4434. * goes backwards.
  4435. *
  4436. * However, we have a problem. We can't atomically update
  4437. * the frequency of a given CPU from this function; it is
  4438. * merely a notifier, which can be called from any CPU.
  4439. * Changing the TSC frequency at arbitrary points in time
  4440. * requires a recomputation of local variables related to
  4441. * the TSC for each VCPU. We must flag these local variables
  4442. * to be updated and be sure the update takes place with the
  4443. * new frequency before any guests proceed.
  4444. *
  4445. * Unfortunately, the combination of hotplug CPU and frequency
  4446. * change creates an intractable locking scenario; the order
  4447. * of when these callouts happen is undefined with respect to
  4448. * CPU hotplug, and they can race with each other. As such,
  4449. * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
  4450. * undefined; you can actually have a CPU frequency change take
  4451. * place in between the computation of X and the setting of the
  4452. * variable. To protect against this problem, all updates of
  4453. * the per_cpu tsc_khz variable are done in an interrupt
  4454. * protected IPI, and all callers wishing to update the value
  4455. * must wait for a synchronous IPI to complete (which is trivial
  4456. * if the caller is on the CPU already). This establishes the
  4457. * necessary total order on variable updates.
  4458. *
  4459. * Note that because a guest time update may take place
  4460. * anytime after the setting of the VCPU's request bit, the
  4461. * correct TSC value must be set before the request. However,
  4462. * to ensure the update actually makes it to any guest which
  4463. * starts running in hardware virtualization between the set
  4464. * and the acquisition of the spinlock, we must also ping the
  4465. * CPU after setting the request bit.
  4466. *
  4467. */
  4468. if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
  4469. return 0;
  4470. if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
  4471. return 0;
  4472. smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
  4473. raw_spin_lock(&kvm_lock);
  4474. list_for_each_entry(kvm, &vm_list, vm_list) {
  4475. kvm_for_each_vcpu(i, vcpu, kvm) {
  4476. if (vcpu->cpu != freq->cpu)
  4477. continue;
  4478. kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
  4479. if (vcpu->cpu != smp_processor_id())
  4480. send_ipi = 1;
  4481. }
  4482. }
  4483. raw_spin_unlock(&kvm_lock);
  4484. if (freq->old < freq->new && send_ipi) {
  4485. /*
  4486. * We upscale the frequency. Must make the guest
  4487. * doesn't see old kvmclock values while running with
  4488. * the new frequency, otherwise we risk the guest sees
  4489. * time go backwards.
  4490. *
  4491. * In case we update the frequency for another cpu
  4492. * (which might be in guest context) send an interrupt
  4493. * to kick the cpu out of guest context. Next time
  4494. * guest context is entered kvmclock will be updated,
  4495. * so the guest will not see stale values.
  4496. */
  4497. smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
  4498. }
  4499. return 0;
  4500. }
  4501. static struct notifier_block kvmclock_cpufreq_notifier_block = {
  4502. .notifier_call = kvmclock_cpufreq_notifier
  4503. };
  4504. static int kvmclock_cpu_notifier(struct notifier_block *nfb,
  4505. unsigned long action, void *hcpu)
  4506. {
  4507. unsigned int cpu = (unsigned long)hcpu;
  4508. switch (action) {
  4509. case CPU_ONLINE:
  4510. case CPU_DOWN_FAILED:
  4511. smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
  4512. break;
  4513. case CPU_DOWN_PREPARE:
  4514. smp_call_function_single(cpu, tsc_bad, NULL, 1);
  4515. break;
  4516. }
  4517. return NOTIFY_OK;
  4518. }
  4519. static struct notifier_block kvmclock_cpu_notifier_block = {
  4520. .notifier_call = kvmclock_cpu_notifier,
  4521. .priority = -INT_MAX
  4522. };
  4523. static void kvm_timer_init(void)
  4524. {
  4525. int cpu;
  4526. max_tsc_khz = tsc_khz;
  4527. register_hotcpu_notifier(&kvmclock_cpu_notifier_block);
  4528. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
  4529. #ifdef CONFIG_CPU_FREQ
  4530. struct cpufreq_policy policy;
  4531. memset(&policy, 0, sizeof(policy));
  4532. cpu = get_cpu();
  4533. cpufreq_get_policy(&policy, cpu);
  4534. if (policy.cpuinfo.max_freq)
  4535. max_tsc_khz = policy.cpuinfo.max_freq;
  4536. put_cpu();
  4537. #endif
  4538. cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
  4539. CPUFREQ_TRANSITION_NOTIFIER);
  4540. }
  4541. pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz);
  4542. for_each_online_cpu(cpu)
  4543. smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
  4544. }
  4545. static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
  4546. int kvm_is_in_guest(void)
  4547. {
  4548. return __this_cpu_read(current_vcpu) != NULL;
  4549. }
  4550. static int kvm_is_user_mode(void)
  4551. {
  4552. int user_mode = 3;
  4553. if (__this_cpu_read(current_vcpu))
  4554. user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
  4555. return user_mode != 0;
  4556. }
  4557. static unsigned long kvm_get_guest_ip(void)
  4558. {
  4559. unsigned long ip = 0;
  4560. if (__this_cpu_read(current_vcpu))
  4561. ip = kvm_rip_read(__this_cpu_read(current_vcpu));
  4562. return ip;
  4563. }
  4564. static struct perf_guest_info_callbacks kvm_guest_cbs = {
  4565. .is_in_guest = kvm_is_in_guest,
  4566. .is_user_mode = kvm_is_user_mode,
  4567. .get_guest_ip = kvm_get_guest_ip,
  4568. };
  4569. void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
  4570. {
  4571. __this_cpu_write(current_vcpu, vcpu);
  4572. }
  4573. EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
  4574. void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
  4575. {
  4576. __this_cpu_write(current_vcpu, NULL);
  4577. }
  4578. EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
  4579. static void kvm_set_mmio_spte_mask(void)
  4580. {
  4581. u64 mask;
  4582. int maxphyaddr = boot_cpu_data.x86_phys_bits;
  4583. /*
  4584. * Set the reserved bits and the present bit of an paging-structure
  4585. * entry to generate page fault with PFER.RSV = 1.
  4586. */
  4587. /* Mask the reserved physical address bits. */
  4588. mask = ((1ull << (51 - maxphyaddr + 1)) - 1) << maxphyaddr;
  4589. /* Bit 62 is always reserved for 32bit host. */
  4590. mask |= 0x3ull << 62;
  4591. /* Set the present bit. */
  4592. mask |= 1ull;
  4593. #ifdef CONFIG_X86_64
  4594. /*
  4595. * If reserved bit is not supported, clear the present bit to disable
  4596. * mmio page fault.
  4597. */
  4598. if (maxphyaddr == 52)
  4599. mask &= ~1ull;
  4600. #endif
  4601. kvm_mmu_set_mmio_spte_mask(mask);
  4602. }
  4603. #ifdef CONFIG_X86_64
  4604. static void pvclock_gtod_update_fn(struct work_struct *work)
  4605. {
  4606. struct kvm *kvm;
  4607. struct kvm_vcpu *vcpu;
  4608. int i;
  4609. raw_spin_lock(&kvm_lock);
  4610. list_for_each_entry(kvm, &vm_list, vm_list)
  4611. kvm_for_each_vcpu(i, vcpu, kvm)
  4612. set_bit(KVM_REQ_MASTERCLOCK_UPDATE, &vcpu->requests);
  4613. atomic_set(&kvm_guest_has_master_clock, 0);
  4614. raw_spin_unlock(&kvm_lock);
  4615. }
  4616. static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);
  4617. /*
  4618. * Notification about pvclock gtod data update.
  4619. */
  4620. static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
  4621. void *priv)
  4622. {
  4623. struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
  4624. struct timekeeper *tk = priv;
  4625. update_pvclock_gtod(tk);
  4626. /* disable master clock if host does not trust, or does not
  4627. * use, TSC clocksource
  4628. */
  4629. if (gtod->clock.vclock_mode != VCLOCK_TSC &&
  4630. atomic_read(&kvm_guest_has_master_clock) != 0)
  4631. queue_work(system_long_wq, &pvclock_gtod_work);
  4632. return 0;
  4633. }
  4634. static struct notifier_block pvclock_gtod_notifier = {
  4635. .notifier_call = pvclock_gtod_notify,
  4636. };
  4637. #endif
  4638. int kvm_arch_init(void *opaque)
  4639. {
  4640. int r;
  4641. struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;
  4642. if (kvm_x86_ops) {
  4643. printk(KERN_ERR "kvm: already loaded the other module\n");
  4644. r = -EEXIST;
  4645. goto out;
  4646. }
  4647. if (!ops->cpu_has_kvm_support()) {
  4648. printk(KERN_ERR "kvm: no hardware support\n");
  4649. r = -EOPNOTSUPP;
  4650. goto out;
  4651. }
  4652. if (ops->disabled_by_bios()) {
  4653. printk(KERN_ERR "kvm: disabled by bios\n");
  4654. r = -EOPNOTSUPP;
  4655. goto out;
  4656. }
  4657. r = -ENOMEM;
  4658. shared_msrs = alloc_percpu(struct kvm_shared_msrs);
  4659. if (!shared_msrs) {
  4660. printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
  4661. goto out;
  4662. }
  4663. r = kvm_mmu_module_init();
  4664. if (r)
  4665. goto out_free_percpu;
  4666. kvm_set_mmio_spte_mask();
  4667. kvm_init_msr_list();
  4668. kvm_x86_ops = ops;
  4669. kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
  4670. PT_DIRTY_MASK, PT64_NX_MASK, 0);
  4671. kvm_timer_init();
  4672. perf_register_guest_info_callbacks(&kvm_guest_cbs);
  4673. if (cpu_has_xsave)
  4674. host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
  4675. kvm_lapic_init();
  4676. #ifdef CONFIG_X86_64
  4677. pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
  4678. #endif
  4679. return 0;
  4680. out_free_percpu:
  4681. free_percpu(shared_msrs);
  4682. out:
  4683. return r;
  4684. }
  4685. void kvm_arch_exit(void)
  4686. {
  4687. perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
  4688. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
  4689. cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
  4690. CPUFREQ_TRANSITION_NOTIFIER);
  4691. unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block);
  4692. #ifdef CONFIG_X86_64
  4693. pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
  4694. #endif
  4695. kvm_x86_ops = NULL;
  4696. kvm_mmu_module_exit();
  4697. free_percpu(shared_msrs);
  4698. }
  4699. int kvm_emulate_halt(struct kvm_vcpu *vcpu)
  4700. {
  4701. ++vcpu->stat.halt_exits;
  4702. if (irqchip_in_kernel(vcpu->kvm)) {
  4703. vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
  4704. return 1;
  4705. } else {
  4706. vcpu->run->exit_reason = KVM_EXIT_HLT;
  4707. return 0;
  4708. }
  4709. }
  4710. EXPORT_SYMBOL_GPL(kvm_emulate_halt);
  4711. int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
  4712. {
  4713. u64 param, ingpa, outgpa, ret;
  4714. uint16_t code, rep_idx, rep_cnt, res = HV_STATUS_SUCCESS, rep_done = 0;
  4715. bool fast, longmode;
  4716. int cs_db, cs_l;
  4717. /*
  4718. * hypercall generates UD from non zero cpl and real mode
  4719. * per HYPER-V spec
  4720. */
  4721. if (kvm_x86_ops->get_cpl(vcpu) != 0 || !is_protmode(vcpu)) {
  4722. kvm_queue_exception(vcpu, UD_VECTOR);
  4723. return 0;
  4724. }
  4725. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  4726. longmode = is_long_mode(vcpu) && cs_l == 1;
  4727. if (!longmode) {
  4728. param = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDX) << 32) |
  4729. (kvm_register_read(vcpu, VCPU_REGS_RAX) & 0xffffffff);
  4730. ingpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RBX) << 32) |
  4731. (kvm_register_read(vcpu, VCPU_REGS_RCX) & 0xffffffff);
  4732. outgpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDI) << 32) |
  4733. (kvm_register_read(vcpu, VCPU_REGS_RSI) & 0xffffffff);
  4734. }
  4735. #ifdef CONFIG_X86_64
  4736. else {
  4737. param = kvm_register_read(vcpu, VCPU_REGS_RCX);
  4738. ingpa = kvm_register_read(vcpu, VCPU_REGS_RDX);
  4739. outgpa = kvm_register_read(vcpu, VCPU_REGS_R8);
  4740. }
  4741. #endif
  4742. code = param & 0xffff;
  4743. fast = (param >> 16) & 0x1;
  4744. rep_cnt = (param >> 32) & 0xfff;
  4745. rep_idx = (param >> 48) & 0xfff;
  4746. trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa);
  4747. switch (code) {
  4748. case HV_X64_HV_NOTIFY_LONG_SPIN_WAIT:
  4749. kvm_vcpu_on_spin(vcpu);
  4750. break;
  4751. default:
  4752. res = HV_STATUS_INVALID_HYPERCALL_CODE;
  4753. break;
  4754. }
  4755. ret = res | (((u64)rep_done & 0xfff) << 32);
  4756. if (longmode) {
  4757. kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
  4758. } else {
  4759. kvm_register_write(vcpu, VCPU_REGS_RDX, ret >> 32);
  4760. kvm_register_write(vcpu, VCPU_REGS_RAX, ret & 0xffffffff);
  4761. }
  4762. return 1;
  4763. }
  4764. int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
  4765. {
  4766. unsigned long nr, a0, a1, a2, a3, ret;
  4767. int r = 1;
  4768. if (kvm_hv_hypercall_enabled(vcpu->kvm))
  4769. return kvm_hv_hypercall(vcpu);
  4770. nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
  4771. a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
  4772. a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
  4773. a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
  4774. a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
  4775. trace_kvm_hypercall(nr, a0, a1, a2, a3);
  4776. if (!is_long_mode(vcpu)) {
  4777. nr &= 0xFFFFFFFF;
  4778. a0 &= 0xFFFFFFFF;
  4779. a1 &= 0xFFFFFFFF;
  4780. a2 &= 0xFFFFFFFF;
  4781. a3 &= 0xFFFFFFFF;
  4782. }
  4783. if (kvm_x86_ops->get_cpl(vcpu) != 0) {
  4784. ret = -KVM_EPERM;
  4785. goto out;
  4786. }
  4787. switch (nr) {
  4788. case KVM_HC_VAPIC_POLL_IRQ:
  4789. ret = 0;
  4790. break;
  4791. default:
  4792. ret = -KVM_ENOSYS;
  4793. break;
  4794. }
  4795. out:
  4796. kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
  4797. ++vcpu->stat.hypercalls;
  4798. return r;
  4799. }
  4800. EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
  4801. static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
  4802. {
  4803. struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
  4804. char instruction[3];
  4805. unsigned long rip = kvm_rip_read(vcpu);
  4806. kvm_x86_ops->patch_hypercall(vcpu, instruction);
  4807. return emulator_write_emulated(ctxt, rip, instruction, 3, NULL);
  4808. }
  4809. /*
  4810. * Check if userspace requested an interrupt window, and that the
  4811. * interrupt window is open.
  4812. *
  4813. * No need to exit to userspace if we already have an interrupt queued.
  4814. */
  4815. static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
  4816. {
  4817. return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
  4818. vcpu->run->request_interrupt_window &&
  4819. kvm_arch_interrupt_allowed(vcpu));
  4820. }
  4821. static void post_kvm_run_save(struct kvm_vcpu *vcpu)
  4822. {
  4823. struct kvm_run *kvm_run = vcpu->run;
  4824. kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
  4825. kvm_run->cr8 = kvm_get_cr8(vcpu);
  4826. kvm_run->apic_base = kvm_get_apic_base(vcpu);
  4827. if (irqchip_in_kernel(vcpu->kvm))
  4828. kvm_run->ready_for_interrupt_injection = 1;
  4829. else
  4830. kvm_run->ready_for_interrupt_injection =
  4831. kvm_arch_interrupt_allowed(vcpu) &&
  4832. !kvm_cpu_has_interrupt(vcpu) &&
  4833. !kvm_event_needs_reinjection(vcpu);
  4834. }
  4835. static int vapic_enter(struct kvm_vcpu *vcpu)
  4836. {
  4837. struct kvm_lapic *apic = vcpu->arch.apic;
  4838. struct page *page;
  4839. if (!apic || !apic->vapic_addr)
  4840. return 0;
  4841. page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
  4842. if (is_error_page(page))
  4843. return -EFAULT;
  4844. vcpu->arch.apic->vapic_page = page;
  4845. return 0;
  4846. }
  4847. static void vapic_exit(struct kvm_vcpu *vcpu)
  4848. {
  4849. struct kvm_lapic *apic = vcpu->arch.apic;
  4850. int idx;
  4851. if (!apic || !apic->vapic_addr)
  4852. return;
  4853. idx = srcu_read_lock(&vcpu->kvm->srcu);
  4854. kvm_release_page_dirty(apic->vapic_page);
  4855. mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
  4856. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  4857. }
  4858. static void update_cr8_intercept(struct kvm_vcpu *vcpu)
  4859. {
  4860. int max_irr, tpr;
  4861. if (!kvm_x86_ops->update_cr8_intercept)
  4862. return;
  4863. if (!vcpu->arch.apic)
  4864. return;
  4865. if (!vcpu->arch.apic->vapic_addr)
  4866. max_irr = kvm_lapic_find_highest_irr(vcpu);
  4867. else
  4868. max_irr = -1;
  4869. if (max_irr != -1)
  4870. max_irr >>= 4;
  4871. tpr = kvm_lapic_get_cr8(vcpu);
  4872. kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
  4873. }
  4874. static void inject_pending_event(struct kvm_vcpu *vcpu)
  4875. {
  4876. /* try to reinject previous events if any */
  4877. if (vcpu->arch.exception.pending) {
  4878. trace_kvm_inj_exception(vcpu->arch.exception.nr,
  4879. vcpu->arch.exception.has_error_code,
  4880. vcpu->arch.exception.error_code);
  4881. kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
  4882. vcpu->arch.exception.has_error_code,
  4883. vcpu->arch.exception.error_code,
  4884. vcpu->arch.exception.reinject);
  4885. return;
  4886. }
  4887. if (vcpu->arch.nmi_injected) {
  4888. kvm_x86_ops->set_nmi(vcpu);
  4889. return;
  4890. }
  4891. if (vcpu->arch.interrupt.pending) {
  4892. kvm_x86_ops->set_irq(vcpu);
  4893. return;
  4894. }
  4895. /* try to inject new event if pending */
  4896. if (vcpu->arch.nmi_pending) {
  4897. if (kvm_x86_ops->nmi_allowed(vcpu)) {
  4898. --vcpu->arch.nmi_pending;
  4899. vcpu->arch.nmi_injected = true;
  4900. kvm_x86_ops->set_nmi(vcpu);
  4901. }
  4902. } else if (kvm_cpu_has_injectable_intr(vcpu)) {
  4903. if (kvm_x86_ops->interrupt_allowed(vcpu)) {
  4904. kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
  4905. false);
  4906. kvm_x86_ops->set_irq(vcpu);
  4907. }
  4908. }
  4909. }
  4910. static void process_nmi(struct kvm_vcpu *vcpu)
  4911. {
  4912. unsigned limit = 2;
  4913. /*
  4914. * x86 is limited to one NMI running, and one NMI pending after it.
  4915. * If an NMI is already in progress, limit further NMIs to just one.
  4916. * Otherwise, allow two (and we'll inject the first one immediately).
  4917. */
  4918. if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
  4919. limit = 1;
  4920. vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
  4921. vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
  4922. kvm_make_request(KVM_REQ_EVENT, vcpu);
  4923. }
  4924. static void kvm_gen_update_masterclock(struct kvm *kvm)
  4925. {
  4926. #ifdef CONFIG_X86_64
  4927. int i;
  4928. struct kvm_vcpu *vcpu;
  4929. struct kvm_arch *ka = &kvm->arch;
  4930. spin_lock(&ka->pvclock_gtod_sync_lock);
  4931. kvm_make_mclock_inprogress_request(kvm);
  4932. /* no guest entries from this point */
  4933. pvclock_update_vm_gtod_copy(kvm);
  4934. kvm_for_each_vcpu(i, vcpu, kvm)
  4935. set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
  4936. /* guest entries allowed */
  4937. kvm_for_each_vcpu(i, vcpu, kvm)
  4938. clear_bit(KVM_REQ_MCLOCK_INPROGRESS, &vcpu->requests);
  4939. spin_unlock(&ka->pvclock_gtod_sync_lock);
  4940. #endif
  4941. }
  4942. static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
  4943. {
  4944. u64 eoi_exit_bitmap[4];
  4945. u32 tmr[8];
  4946. if (!kvm_apic_hw_enabled(vcpu->arch.apic))
  4947. return;
  4948. memset(eoi_exit_bitmap, 0, 32);
  4949. memset(tmr, 0, 32);
  4950. kvm_ioapic_scan_entry(vcpu, eoi_exit_bitmap, tmr);
  4951. kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
  4952. kvm_apic_update_tmr(vcpu, tmr);
  4953. }
  4954. static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
  4955. {
  4956. int r;
  4957. bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
  4958. vcpu->run->request_interrupt_window;
  4959. bool req_immediate_exit = false;
  4960. if (vcpu->requests) {
  4961. if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
  4962. kvm_mmu_unload(vcpu);
  4963. if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
  4964. __kvm_migrate_timers(vcpu);
  4965. if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
  4966. kvm_gen_update_masterclock(vcpu->kvm);
  4967. if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
  4968. kvm_gen_kvmclock_update(vcpu);
  4969. if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
  4970. r = kvm_guest_time_update(vcpu);
  4971. if (unlikely(r))
  4972. goto out;
  4973. }
  4974. if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
  4975. kvm_mmu_sync_roots(vcpu);
  4976. if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
  4977. kvm_x86_ops->tlb_flush(vcpu);
  4978. if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
  4979. vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
  4980. r = 0;
  4981. goto out;
  4982. }
  4983. if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
  4984. vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
  4985. r = 0;
  4986. goto out;
  4987. }
  4988. if (kvm_check_request(KVM_REQ_DEACTIVATE_FPU, vcpu)) {
  4989. vcpu->fpu_active = 0;
  4990. kvm_x86_ops->fpu_deactivate(vcpu);
  4991. }
  4992. if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
  4993. /* Page is swapped out. Do synthetic halt */
  4994. vcpu->arch.apf.halted = true;
  4995. r = 1;
  4996. goto out;
  4997. }
  4998. if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
  4999. record_steal_time(vcpu);
  5000. if (kvm_check_request(KVM_REQ_NMI, vcpu))
  5001. process_nmi(vcpu);
  5002. if (kvm_check_request(KVM_REQ_PMU, vcpu))
  5003. kvm_handle_pmu_event(vcpu);
  5004. if (kvm_check_request(KVM_REQ_PMI, vcpu))
  5005. kvm_deliver_pmi(vcpu);
  5006. if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
  5007. vcpu_scan_ioapic(vcpu);
  5008. }
  5009. if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
  5010. kvm_apic_accept_events(vcpu);
  5011. if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
  5012. r = 1;
  5013. goto out;
  5014. }
  5015. inject_pending_event(vcpu);
  5016. /* enable NMI/IRQ window open exits if needed */
  5017. if (vcpu->arch.nmi_pending)
  5018. req_immediate_exit =
  5019. kvm_x86_ops->enable_nmi_window(vcpu) != 0;
  5020. else if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
  5021. req_immediate_exit =
  5022. kvm_x86_ops->enable_irq_window(vcpu) != 0;
  5023. if (kvm_lapic_enabled(vcpu)) {
  5024. /*
  5025. * Update architecture specific hints for APIC
  5026. * virtual interrupt delivery.
  5027. */
  5028. if (kvm_x86_ops->hwapic_irr_update)
  5029. kvm_x86_ops->hwapic_irr_update(vcpu,
  5030. kvm_lapic_find_highest_irr(vcpu));
  5031. update_cr8_intercept(vcpu);
  5032. kvm_lapic_sync_to_vapic(vcpu);
  5033. }
  5034. }
  5035. r = kvm_mmu_reload(vcpu);
  5036. if (unlikely(r)) {
  5037. goto cancel_injection;
  5038. }
  5039. preempt_disable();
  5040. kvm_x86_ops->prepare_guest_switch(vcpu);
  5041. if (vcpu->fpu_active)
  5042. kvm_load_guest_fpu(vcpu);
  5043. kvm_load_guest_xcr0(vcpu);
  5044. vcpu->mode = IN_GUEST_MODE;
  5045. /* We should set ->mode before check ->requests,
  5046. * see the comment in make_all_cpus_request.
  5047. */
  5048. smp_mb();
  5049. local_irq_disable();
  5050. if (vcpu->mode == EXITING_GUEST_MODE || vcpu->requests
  5051. || need_resched() || signal_pending(current)) {
  5052. vcpu->mode = OUTSIDE_GUEST_MODE;
  5053. smp_wmb();
  5054. local_irq_enable();
  5055. preempt_enable();
  5056. r = 1;
  5057. goto cancel_injection;
  5058. }
  5059. srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
  5060. if (req_immediate_exit)
  5061. smp_send_reschedule(vcpu->cpu);
  5062. kvm_guest_enter();
  5063. if (unlikely(vcpu->arch.switch_db_regs)) {
  5064. set_debugreg(0, 7);
  5065. set_debugreg(vcpu->arch.eff_db[0], 0);
  5066. set_debugreg(vcpu->arch.eff_db[1], 1);
  5067. set_debugreg(vcpu->arch.eff_db[2], 2);
  5068. set_debugreg(vcpu->arch.eff_db[3], 3);
  5069. }
  5070. trace_kvm_entry(vcpu->vcpu_id);
  5071. kvm_x86_ops->run(vcpu);
  5072. /*
  5073. * If the guest has used debug registers, at least dr7
  5074. * will be disabled while returning to the host.
  5075. * If we don't have active breakpoints in the host, we don't
  5076. * care about the messed up debug address registers. But if
  5077. * we have some of them active, restore the old state.
  5078. */
  5079. if (hw_breakpoint_active())
  5080. hw_breakpoint_restore();
  5081. vcpu->arch.last_guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu,
  5082. native_read_tsc());
  5083. vcpu->mode = OUTSIDE_GUEST_MODE;
  5084. smp_wmb();
  5085. /* Interrupt is enabled by handle_external_intr() */
  5086. kvm_x86_ops->handle_external_intr(vcpu);
  5087. ++vcpu->stat.exits;
  5088. /*
  5089. * We must have an instruction between local_irq_enable() and
  5090. * kvm_guest_exit(), so the timer interrupt isn't delayed by
  5091. * the interrupt shadow. The stat.exits increment will do nicely.
  5092. * But we need to prevent reordering, hence this barrier():
  5093. */
  5094. barrier();
  5095. kvm_guest_exit();
  5096. preempt_enable();
  5097. vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  5098. /*
  5099. * Profile KVM exit RIPs:
  5100. */
  5101. if (unlikely(prof_on == KVM_PROFILING)) {
  5102. unsigned long rip = kvm_rip_read(vcpu);
  5103. profile_hit(KVM_PROFILING, (void *)rip);
  5104. }
  5105. if (unlikely(vcpu->arch.tsc_always_catchup))
  5106. kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
  5107. if (vcpu->arch.apic_attention)
  5108. kvm_lapic_sync_from_vapic(vcpu);
  5109. r = kvm_x86_ops->handle_exit(vcpu);
  5110. return r;
  5111. cancel_injection:
  5112. kvm_x86_ops->cancel_injection(vcpu);
  5113. if (unlikely(vcpu->arch.apic_attention))
  5114. kvm_lapic_sync_from_vapic(vcpu);
  5115. out:
  5116. return r;
  5117. }
  5118. static int __vcpu_run(struct kvm_vcpu *vcpu)
  5119. {
  5120. int r;
  5121. struct kvm *kvm = vcpu->kvm;
  5122. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  5123. r = vapic_enter(vcpu);
  5124. if (r) {
  5125. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  5126. return r;
  5127. }
  5128. r = 1;
  5129. while (r > 0) {
  5130. if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
  5131. !vcpu->arch.apf.halted)
  5132. r = vcpu_enter_guest(vcpu);
  5133. else {
  5134. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  5135. kvm_vcpu_block(vcpu);
  5136. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  5137. if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
  5138. kvm_apic_accept_events(vcpu);
  5139. switch(vcpu->arch.mp_state) {
  5140. case KVM_MP_STATE_HALTED:
  5141. vcpu->arch.mp_state =
  5142. KVM_MP_STATE_RUNNABLE;
  5143. case KVM_MP_STATE_RUNNABLE:
  5144. vcpu->arch.apf.halted = false;
  5145. break;
  5146. case KVM_MP_STATE_INIT_RECEIVED:
  5147. break;
  5148. default:
  5149. r = -EINTR;
  5150. break;
  5151. }
  5152. }
  5153. }
  5154. if (r <= 0)
  5155. break;
  5156. clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
  5157. if (kvm_cpu_has_pending_timer(vcpu))
  5158. kvm_inject_pending_timer_irqs(vcpu);
  5159. if (dm_request_for_irq_injection(vcpu)) {
  5160. r = -EINTR;
  5161. vcpu->run->exit_reason = KVM_EXIT_INTR;
  5162. ++vcpu->stat.request_irq_exits;
  5163. }
  5164. kvm_check_async_pf_completion(vcpu);
  5165. if (signal_pending(current)) {
  5166. r = -EINTR;
  5167. vcpu->run->exit_reason = KVM_EXIT_INTR;
  5168. ++vcpu->stat.signal_exits;
  5169. }
  5170. if (need_resched()) {
  5171. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  5172. kvm_resched(vcpu);
  5173. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  5174. }
  5175. }
  5176. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  5177. vapic_exit(vcpu);
  5178. return r;
  5179. }
  5180. static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
  5181. {
  5182. int r;
  5183. vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  5184. r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
  5185. srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
  5186. if (r != EMULATE_DONE)
  5187. return 0;
  5188. return 1;
  5189. }
  5190. static int complete_emulated_pio(struct kvm_vcpu *vcpu)
  5191. {
  5192. BUG_ON(!vcpu->arch.pio.count);
  5193. return complete_emulated_io(vcpu);
  5194. }
  5195. /*
  5196. * Implements the following, as a state machine:
  5197. *
  5198. * read:
  5199. * for each fragment
  5200. * for each mmio piece in the fragment
  5201. * write gpa, len
  5202. * exit
  5203. * copy data
  5204. * execute insn
  5205. *
  5206. * write:
  5207. * for each fragment
  5208. * for each mmio piece in the fragment
  5209. * write gpa, len
  5210. * copy data
  5211. * exit
  5212. */
  5213. static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
  5214. {
  5215. struct kvm_run *run = vcpu->run;
  5216. struct kvm_mmio_fragment *frag;
  5217. unsigned len;
  5218. BUG_ON(!vcpu->mmio_needed);
  5219. /* Complete previous fragment */
  5220. frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
  5221. len = min(8u, frag->len);
  5222. if (!vcpu->mmio_is_write)
  5223. memcpy(frag->data, run->mmio.data, len);
  5224. if (frag->len <= 8) {
  5225. /* Switch to the next fragment. */
  5226. frag++;
  5227. vcpu->mmio_cur_fragment++;
  5228. } else {
  5229. /* Go forward to the next mmio piece. */
  5230. frag->data += len;
  5231. frag->gpa += len;
  5232. frag->len -= len;
  5233. }
  5234. if (vcpu->mmio_cur_fragment == vcpu->mmio_nr_fragments) {
  5235. vcpu->mmio_needed = 0;
  5236. if (vcpu->mmio_is_write)
  5237. return 1;
  5238. vcpu->mmio_read_completed = 1;
  5239. return complete_emulated_io(vcpu);
  5240. }
  5241. run->exit_reason = KVM_EXIT_MMIO;
  5242. run->mmio.phys_addr = frag->gpa;
  5243. if (vcpu->mmio_is_write)
  5244. memcpy(run->mmio.data, frag->data, min(8u, frag->len));
  5245. run->mmio.len = min(8u, frag->len);
  5246. run->mmio.is_write = vcpu->mmio_is_write;
  5247. vcpu->arch.complete_userspace_io = complete_emulated_mmio;
  5248. return 0;
  5249. }
  5250. int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  5251. {
  5252. int r;
  5253. sigset_t sigsaved;
  5254. if (!tsk_used_math(current) && init_fpu(current))
  5255. return -ENOMEM;
  5256. if (vcpu->sigset_active)
  5257. sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
  5258. if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
  5259. kvm_vcpu_block(vcpu);
  5260. kvm_apic_accept_events(vcpu);
  5261. clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
  5262. r = -EAGAIN;
  5263. goto out;
  5264. }
  5265. /* re-sync apic's tpr */
  5266. if (!irqchip_in_kernel(vcpu->kvm)) {
  5267. if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
  5268. r = -EINVAL;
  5269. goto out;
  5270. }
  5271. }
  5272. if (unlikely(vcpu->arch.complete_userspace_io)) {
  5273. int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
  5274. vcpu->arch.complete_userspace_io = NULL;
  5275. r = cui(vcpu);
  5276. if (r <= 0)
  5277. goto out;
  5278. } else
  5279. WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
  5280. r = __vcpu_run(vcpu);
  5281. out:
  5282. post_kvm_run_save(vcpu);
  5283. if (vcpu->sigset_active)
  5284. sigprocmask(SIG_SETMASK, &sigsaved, NULL);
  5285. return r;
  5286. }
  5287. int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  5288. {
  5289. if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
  5290. /*
  5291. * We are here if userspace calls get_regs() in the middle of
  5292. * instruction emulation. Registers state needs to be copied
  5293. * back from emulation context to vcpu. Userspace shouldn't do
  5294. * that usually, but some bad designed PV devices (vmware
  5295. * backdoor interface) need this to work
  5296. */
  5297. emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
  5298. vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
  5299. }
  5300. regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  5301. regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  5302. regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  5303. regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  5304. regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
  5305. regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
  5306. regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  5307. regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  5308. #ifdef CONFIG_X86_64
  5309. regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
  5310. regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
  5311. regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
  5312. regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
  5313. regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
  5314. regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
  5315. regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
  5316. regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
  5317. #endif
  5318. regs->rip = kvm_rip_read(vcpu);
  5319. regs->rflags = kvm_get_rflags(vcpu);
  5320. return 0;
  5321. }
  5322. int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  5323. {
  5324. vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
  5325. vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
  5326. kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
  5327. kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
  5328. kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
  5329. kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
  5330. kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
  5331. kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
  5332. kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
  5333. kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
  5334. #ifdef CONFIG_X86_64
  5335. kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
  5336. kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
  5337. kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
  5338. kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
  5339. kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
  5340. kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
  5341. kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
  5342. kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
  5343. #endif
  5344. kvm_rip_write(vcpu, regs->rip);
  5345. kvm_set_rflags(vcpu, regs->rflags);
  5346. vcpu->arch.exception.pending = false;
  5347. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5348. return 0;
  5349. }
  5350. void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
  5351. {
  5352. struct kvm_segment cs;
  5353. kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
  5354. *db = cs.db;
  5355. *l = cs.l;
  5356. }
  5357. EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
  5358. int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  5359. struct kvm_sregs *sregs)
  5360. {
  5361. struct desc_ptr dt;
  5362. kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  5363. kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  5364. kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  5365. kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  5366. kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  5367. kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  5368. kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  5369. kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  5370. kvm_x86_ops->get_idt(vcpu, &dt);
  5371. sregs->idt.limit = dt.size;
  5372. sregs->idt.base = dt.address;
  5373. kvm_x86_ops->get_gdt(vcpu, &dt);
  5374. sregs->gdt.limit = dt.size;
  5375. sregs->gdt.base = dt.address;
  5376. sregs->cr0 = kvm_read_cr0(vcpu);
  5377. sregs->cr2 = vcpu->arch.cr2;
  5378. sregs->cr3 = kvm_read_cr3(vcpu);
  5379. sregs->cr4 = kvm_read_cr4(vcpu);
  5380. sregs->cr8 = kvm_get_cr8(vcpu);
  5381. sregs->efer = vcpu->arch.efer;
  5382. sregs->apic_base = kvm_get_apic_base(vcpu);
  5383. memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
  5384. if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
  5385. set_bit(vcpu->arch.interrupt.nr,
  5386. (unsigned long *)sregs->interrupt_bitmap);
  5387. return 0;
  5388. }
  5389. int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
  5390. struct kvm_mp_state *mp_state)
  5391. {
  5392. kvm_apic_accept_events(vcpu);
  5393. mp_state->mp_state = vcpu->arch.mp_state;
  5394. return 0;
  5395. }
  5396. int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
  5397. struct kvm_mp_state *mp_state)
  5398. {
  5399. if (!kvm_vcpu_has_lapic(vcpu) &&
  5400. mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
  5401. return -EINVAL;
  5402. if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
  5403. vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
  5404. set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
  5405. } else
  5406. vcpu->arch.mp_state = mp_state->mp_state;
  5407. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5408. return 0;
  5409. }
  5410. int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
  5411. int reason, bool has_error_code, u32 error_code)
  5412. {
  5413. struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
  5414. int ret;
  5415. init_emulate_ctxt(vcpu);
  5416. ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
  5417. has_error_code, error_code);
  5418. if (ret)
  5419. return EMULATE_FAIL;
  5420. kvm_rip_write(vcpu, ctxt->eip);
  5421. kvm_set_rflags(vcpu, ctxt->eflags);
  5422. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5423. return EMULATE_DONE;
  5424. }
  5425. EXPORT_SYMBOL_GPL(kvm_task_switch);
  5426. int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  5427. struct kvm_sregs *sregs)
  5428. {
  5429. int mmu_reset_needed = 0;
  5430. int pending_vec, max_bits, idx;
  5431. struct desc_ptr dt;
  5432. if (!guest_cpuid_has_xsave(vcpu) && (sregs->cr4 & X86_CR4_OSXSAVE))
  5433. return -EINVAL;
  5434. dt.size = sregs->idt.limit;
  5435. dt.address = sregs->idt.base;
  5436. kvm_x86_ops->set_idt(vcpu, &dt);
  5437. dt.size = sregs->gdt.limit;
  5438. dt.address = sregs->gdt.base;
  5439. kvm_x86_ops->set_gdt(vcpu, &dt);
  5440. vcpu->arch.cr2 = sregs->cr2;
  5441. mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
  5442. vcpu->arch.cr3 = sregs->cr3;
  5443. __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
  5444. kvm_set_cr8(vcpu, sregs->cr8);
  5445. mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
  5446. kvm_x86_ops->set_efer(vcpu, sregs->efer);
  5447. kvm_set_apic_base(vcpu, sregs->apic_base);
  5448. mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
  5449. kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
  5450. vcpu->arch.cr0 = sregs->cr0;
  5451. mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
  5452. kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
  5453. if (sregs->cr4 & X86_CR4_OSXSAVE)
  5454. kvm_update_cpuid(vcpu);
  5455. idx = srcu_read_lock(&vcpu->kvm->srcu);
  5456. if (!is_long_mode(vcpu) && is_pae(vcpu)) {
  5457. load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
  5458. mmu_reset_needed = 1;
  5459. }
  5460. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  5461. if (mmu_reset_needed)
  5462. kvm_mmu_reset_context(vcpu);
  5463. max_bits = KVM_NR_INTERRUPTS;
  5464. pending_vec = find_first_bit(
  5465. (const unsigned long *)sregs->interrupt_bitmap, max_bits);
  5466. if (pending_vec < max_bits) {
  5467. kvm_queue_interrupt(vcpu, pending_vec, false);
  5468. pr_debug("Set back pending irq %d\n", pending_vec);
  5469. }
  5470. kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  5471. kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  5472. kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  5473. kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  5474. kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  5475. kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  5476. kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  5477. kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  5478. update_cr8_intercept(vcpu);
  5479. /* Older userspace won't unhalt the vcpu on reset. */
  5480. if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
  5481. sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
  5482. !is_protmode(vcpu))
  5483. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  5484. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5485. return 0;
  5486. }
  5487. int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
  5488. struct kvm_guest_debug *dbg)
  5489. {
  5490. unsigned long rflags;
  5491. int i, r;
  5492. if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
  5493. r = -EBUSY;
  5494. if (vcpu->arch.exception.pending)
  5495. goto out;
  5496. if (dbg->control & KVM_GUESTDBG_INJECT_DB)
  5497. kvm_queue_exception(vcpu, DB_VECTOR);
  5498. else
  5499. kvm_queue_exception(vcpu, BP_VECTOR);
  5500. }
  5501. /*
  5502. * Read rflags as long as potentially injected trace flags are still
  5503. * filtered out.
  5504. */
  5505. rflags = kvm_get_rflags(vcpu);
  5506. vcpu->guest_debug = dbg->control;
  5507. if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
  5508. vcpu->guest_debug = 0;
  5509. if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
  5510. for (i = 0; i < KVM_NR_DB_REGS; ++i)
  5511. vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
  5512. vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
  5513. } else {
  5514. for (i = 0; i < KVM_NR_DB_REGS; i++)
  5515. vcpu->arch.eff_db[i] = vcpu->arch.db[i];
  5516. }
  5517. kvm_update_dr7(vcpu);
  5518. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
  5519. vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
  5520. get_segment_base(vcpu, VCPU_SREG_CS);
  5521. /*
  5522. * Trigger an rflags update that will inject or remove the trace
  5523. * flags.
  5524. */
  5525. kvm_set_rflags(vcpu, rflags);
  5526. kvm_x86_ops->update_db_bp_intercept(vcpu);
  5527. r = 0;
  5528. out:
  5529. return r;
  5530. }
  5531. /*
  5532. * Translate a guest virtual address to a guest physical address.
  5533. */
  5534. int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  5535. struct kvm_translation *tr)
  5536. {
  5537. unsigned long vaddr = tr->linear_address;
  5538. gpa_t gpa;
  5539. int idx;
  5540. idx = srcu_read_lock(&vcpu->kvm->srcu);
  5541. gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
  5542. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  5543. tr->physical_address = gpa;
  5544. tr->valid = gpa != UNMAPPED_GVA;
  5545. tr->writeable = 1;
  5546. tr->usermode = 0;
  5547. return 0;
  5548. }
  5549. int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  5550. {
  5551. struct i387_fxsave_struct *fxsave =
  5552. &vcpu->arch.guest_fpu.state->fxsave;
  5553. memcpy(fpu->fpr, fxsave->st_space, 128);
  5554. fpu->fcw = fxsave->cwd;
  5555. fpu->fsw = fxsave->swd;
  5556. fpu->ftwx = fxsave->twd;
  5557. fpu->last_opcode = fxsave->fop;
  5558. fpu->last_ip = fxsave->rip;
  5559. fpu->last_dp = fxsave->rdp;
  5560. memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
  5561. return 0;
  5562. }
  5563. int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  5564. {
  5565. struct i387_fxsave_struct *fxsave =
  5566. &vcpu->arch.guest_fpu.state->fxsave;
  5567. memcpy(fxsave->st_space, fpu->fpr, 128);
  5568. fxsave->cwd = fpu->fcw;
  5569. fxsave->swd = fpu->fsw;
  5570. fxsave->twd = fpu->ftwx;
  5571. fxsave->fop = fpu->last_opcode;
  5572. fxsave->rip = fpu->last_ip;
  5573. fxsave->rdp = fpu->last_dp;
  5574. memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
  5575. return 0;
  5576. }
  5577. int fx_init(struct kvm_vcpu *vcpu)
  5578. {
  5579. int err;
  5580. err = fpu_alloc(&vcpu->arch.guest_fpu);
  5581. if (err)
  5582. return err;
  5583. fpu_finit(&vcpu->arch.guest_fpu);
  5584. /*
  5585. * Ensure guest xcr0 is valid for loading
  5586. */
  5587. vcpu->arch.xcr0 = XSTATE_FP;
  5588. vcpu->arch.cr0 |= X86_CR0_ET;
  5589. return 0;
  5590. }
  5591. EXPORT_SYMBOL_GPL(fx_init);
  5592. static void fx_free(struct kvm_vcpu *vcpu)
  5593. {
  5594. fpu_free(&vcpu->arch.guest_fpu);
  5595. }
  5596. void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
  5597. {
  5598. if (vcpu->guest_fpu_loaded)
  5599. return;
  5600. /*
  5601. * Restore all possible states in the guest,
  5602. * and assume host would use all available bits.
  5603. * Guest xcr0 would be loaded later.
  5604. */
  5605. kvm_put_guest_xcr0(vcpu);
  5606. vcpu->guest_fpu_loaded = 1;
  5607. __kernel_fpu_begin();
  5608. fpu_restore_checking(&vcpu->arch.guest_fpu);
  5609. trace_kvm_fpu(1);
  5610. }
  5611. void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
  5612. {
  5613. kvm_put_guest_xcr0(vcpu);
  5614. if (!vcpu->guest_fpu_loaded)
  5615. return;
  5616. vcpu->guest_fpu_loaded = 0;
  5617. fpu_save_init(&vcpu->arch.guest_fpu);
  5618. __kernel_fpu_end();
  5619. ++vcpu->stat.fpu_reload;
  5620. kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu);
  5621. trace_kvm_fpu(0);
  5622. }
  5623. void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
  5624. {
  5625. kvmclock_reset(vcpu);
  5626. free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
  5627. fx_free(vcpu);
  5628. kvm_x86_ops->vcpu_free(vcpu);
  5629. }
  5630. struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
  5631. unsigned int id)
  5632. {
  5633. if (check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
  5634. printk_once(KERN_WARNING
  5635. "kvm: SMP vm created on host with unstable TSC; "
  5636. "guest TSC will not be reliable\n");
  5637. return kvm_x86_ops->vcpu_create(kvm, id);
  5638. }
  5639. int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
  5640. {
  5641. int r;
  5642. vcpu->arch.mtrr_state.have_fixed = 1;
  5643. r = vcpu_load(vcpu);
  5644. if (r)
  5645. return r;
  5646. kvm_vcpu_reset(vcpu);
  5647. r = kvm_mmu_setup(vcpu);
  5648. vcpu_put(vcpu);
  5649. return r;
  5650. }
  5651. int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
  5652. {
  5653. int r;
  5654. struct msr_data msr;
  5655. r = vcpu_load(vcpu);
  5656. if (r)
  5657. return r;
  5658. msr.data = 0x0;
  5659. msr.index = MSR_IA32_TSC;
  5660. msr.host_initiated = true;
  5661. kvm_write_tsc(vcpu, &msr);
  5662. vcpu_put(vcpu);
  5663. return r;
  5664. }
  5665. void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
  5666. {
  5667. int r;
  5668. vcpu->arch.apf.msr_val = 0;
  5669. r = vcpu_load(vcpu);
  5670. BUG_ON(r);
  5671. kvm_mmu_unload(vcpu);
  5672. vcpu_put(vcpu);
  5673. fx_free(vcpu);
  5674. kvm_x86_ops->vcpu_free(vcpu);
  5675. }
  5676. void kvm_vcpu_reset(struct kvm_vcpu *vcpu)
  5677. {
  5678. atomic_set(&vcpu->arch.nmi_queued, 0);
  5679. vcpu->arch.nmi_pending = 0;
  5680. vcpu->arch.nmi_injected = false;
  5681. memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
  5682. vcpu->arch.dr6 = DR6_FIXED_1;
  5683. vcpu->arch.dr7 = DR7_FIXED_1;
  5684. kvm_update_dr7(vcpu);
  5685. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5686. vcpu->arch.apf.msr_val = 0;
  5687. vcpu->arch.st.msr_val = 0;
  5688. kvmclock_reset(vcpu);
  5689. kvm_clear_async_pf_completion_queue(vcpu);
  5690. kvm_async_pf_hash_reset(vcpu);
  5691. vcpu->arch.apf.halted = false;
  5692. kvm_pmu_reset(vcpu);
  5693. memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
  5694. vcpu->arch.regs_avail = ~0;
  5695. vcpu->arch.regs_dirty = ~0;
  5696. kvm_x86_ops->vcpu_reset(vcpu);
  5697. }
  5698. void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, unsigned int vector)
  5699. {
  5700. struct kvm_segment cs;
  5701. kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
  5702. cs.selector = vector << 8;
  5703. cs.base = vector << 12;
  5704. kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
  5705. kvm_rip_write(vcpu, 0);
  5706. }
  5707. int kvm_arch_hardware_enable(void *garbage)
  5708. {
  5709. struct kvm *kvm;
  5710. struct kvm_vcpu *vcpu;
  5711. int i;
  5712. int ret;
  5713. u64 local_tsc;
  5714. u64 max_tsc = 0;
  5715. bool stable, backwards_tsc = false;
  5716. kvm_shared_msr_cpu_online();
  5717. ret = kvm_x86_ops->hardware_enable(garbage);
  5718. if (ret != 0)
  5719. return ret;
  5720. local_tsc = native_read_tsc();
  5721. stable = !check_tsc_unstable();
  5722. list_for_each_entry(kvm, &vm_list, vm_list) {
  5723. kvm_for_each_vcpu(i, vcpu, kvm) {
  5724. if (!stable && vcpu->cpu == smp_processor_id())
  5725. set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
  5726. if (stable && vcpu->arch.last_host_tsc > local_tsc) {
  5727. backwards_tsc = true;
  5728. if (vcpu->arch.last_host_tsc > max_tsc)
  5729. max_tsc = vcpu->arch.last_host_tsc;
  5730. }
  5731. }
  5732. }
  5733. /*
  5734. * Sometimes, even reliable TSCs go backwards. This happens on
  5735. * platforms that reset TSC during suspend or hibernate actions, but
  5736. * maintain synchronization. We must compensate. Fortunately, we can
  5737. * detect that condition here, which happens early in CPU bringup,
  5738. * before any KVM threads can be running. Unfortunately, we can't
  5739. * bring the TSCs fully up to date with real time, as we aren't yet far
  5740. * enough into CPU bringup that we know how much real time has actually
  5741. * elapsed; our helper function, get_kernel_ns() will be using boot
  5742. * variables that haven't been updated yet.
  5743. *
  5744. * So we simply find the maximum observed TSC above, then record the
  5745. * adjustment to TSC in each VCPU. When the VCPU later gets loaded,
  5746. * the adjustment will be applied. Note that we accumulate
  5747. * adjustments, in case multiple suspend cycles happen before some VCPU
  5748. * gets a chance to run again. In the event that no KVM threads get a
  5749. * chance to run, we will miss the entire elapsed period, as we'll have
  5750. * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
  5751. * loose cycle time. This isn't too big a deal, since the loss will be
  5752. * uniform across all VCPUs (not to mention the scenario is extremely
  5753. * unlikely). It is possible that a second hibernate recovery happens
  5754. * much faster than a first, causing the observed TSC here to be
  5755. * smaller; this would require additional padding adjustment, which is
  5756. * why we set last_host_tsc to the local tsc observed here.
  5757. *
  5758. * N.B. - this code below runs only on platforms with reliable TSC,
  5759. * as that is the only way backwards_tsc is set above. Also note
  5760. * that this runs for ALL vcpus, which is not a bug; all VCPUs should
  5761. * have the same delta_cyc adjustment applied if backwards_tsc
  5762. * is detected. Note further, this adjustment is only done once,
  5763. * as we reset last_host_tsc on all VCPUs to stop this from being
  5764. * called multiple times (one for each physical CPU bringup).
  5765. *
  5766. * Platforms with unreliable TSCs don't have to deal with this, they
  5767. * will be compensated by the logic in vcpu_load, which sets the TSC to
  5768. * catchup mode. This will catchup all VCPUs to real time, but cannot
  5769. * guarantee that they stay in perfect synchronization.
  5770. */
  5771. if (backwards_tsc) {
  5772. u64 delta_cyc = max_tsc - local_tsc;
  5773. list_for_each_entry(kvm, &vm_list, vm_list) {
  5774. kvm_for_each_vcpu(i, vcpu, kvm) {
  5775. vcpu->arch.tsc_offset_adjustment += delta_cyc;
  5776. vcpu->arch.last_host_tsc = local_tsc;
  5777. set_bit(KVM_REQ_MASTERCLOCK_UPDATE,
  5778. &vcpu->requests);
  5779. }
  5780. /*
  5781. * We have to disable TSC offset matching.. if you were
  5782. * booting a VM while issuing an S4 host suspend....
  5783. * you may have some problem. Solving this issue is
  5784. * left as an exercise to the reader.
  5785. */
  5786. kvm->arch.last_tsc_nsec = 0;
  5787. kvm->arch.last_tsc_write = 0;
  5788. }
  5789. }
  5790. return 0;
  5791. }
  5792. void kvm_arch_hardware_disable(void *garbage)
  5793. {
  5794. kvm_x86_ops->hardware_disable(garbage);
  5795. drop_user_return_notifiers(garbage);
  5796. }
  5797. int kvm_arch_hardware_setup(void)
  5798. {
  5799. return kvm_x86_ops->hardware_setup();
  5800. }
  5801. void kvm_arch_hardware_unsetup(void)
  5802. {
  5803. kvm_x86_ops->hardware_unsetup();
  5804. }
  5805. void kvm_arch_check_processor_compat(void *rtn)
  5806. {
  5807. kvm_x86_ops->check_processor_compatibility(rtn);
  5808. }
  5809. bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu)
  5810. {
  5811. return irqchip_in_kernel(vcpu->kvm) == (vcpu->arch.apic != NULL);
  5812. }
  5813. struct static_key kvm_no_apic_vcpu __read_mostly;
  5814. int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
  5815. {
  5816. struct page *page;
  5817. struct kvm *kvm;
  5818. int r;
  5819. BUG_ON(vcpu->kvm == NULL);
  5820. kvm = vcpu->kvm;
  5821. vcpu->arch.emulate_ctxt.ops = &emulate_ops;
  5822. if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
  5823. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  5824. else
  5825. vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
  5826. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  5827. if (!page) {
  5828. r = -ENOMEM;
  5829. goto fail;
  5830. }
  5831. vcpu->arch.pio_data = page_address(page);
  5832. kvm_set_tsc_khz(vcpu, max_tsc_khz);
  5833. r = kvm_mmu_create(vcpu);
  5834. if (r < 0)
  5835. goto fail_free_pio_data;
  5836. if (irqchip_in_kernel(kvm)) {
  5837. r = kvm_create_lapic(vcpu);
  5838. if (r < 0)
  5839. goto fail_mmu_destroy;
  5840. } else
  5841. static_key_slow_inc(&kvm_no_apic_vcpu);
  5842. vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
  5843. GFP_KERNEL);
  5844. if (!vcpu->arch.mce_banks) {
  5845. r = -ENOMEM;
  5846. goto fail_free_lapic;
  5847. }
  5848. vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
  5849. if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL)) {
  5850. r = -ENOMEM;
  5851. goto fail_free_mce_banks;
  5852. }
  5853. r = fx_init(vcpu);
  5854. if (r)
  5855. goto fail_free_wbinvd_dirty_mask;
  5856. vcpu->arch.ia32_tsc_adjust_msr = 0x0;
  5857. vcpu->arch.pv_time_enabled = false;
  5858. kvm_async_pf_hash_reset(vcpu);
  5859. kvm_pmu_init(vcpu);
  5860. return 0;
  5861. fail_free_wbinvd_dirty_mask:
  5862. free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
  5863. fail_free_mce_banks:
  5864. kfree(vcpu->arch.mce_banks);
  5865. fail_free_lapic:
  5866. kvm_free_lapic(vcpu);
  5867. fail_mmu_destroy:
  5868. kvm_mmu_destroy(vcpu);
  5869. fail_free_pio_data:
  5870. free_page((unsigned long)vcpu->arch.pio_data);
  5871. fail:
  5872. return r;
  5873. }
  5874. void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
  5875. {
  5876. int idx;
  5877. kvm_pmu_destroy(vcpu);
  5878. kfree(vcpu->arch.mce_banks);
  5879. kvm_free_lapic(vcpu);
  5880. idx = srcu_read_lock(&vcpu->kvm->srcu);
  5881. kvm_mmu_destroy(vcpu);
  5882. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  5883. free_page((unsigned long)vcpu->arch.pio_data);
  5884. if (!irqchip_in_kernel(vcpu->kvm))
  5885. static_key_slow_dec(&kvm_no_apic_vcpu);
  5886. }
  5887. int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
  5888. {
  5889. if (type)
  5890. return -EINVAL;
  5891. INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
  5892. INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
  5893. INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
  5894. /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
  5895. set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
  5896. /* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
  5897. set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
  5898. &kvm->arch.irq_sources_bitmap);
  5899. raw_spin_lock_init(&kvm->arch.tsc_write_lock);
  5900. mutex_init(&kvm->arch.apic_map_lock);
  5901. spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);
  5902. pvclock_update_vm_gtod_copy(kvm);
  5903. return 0;
  5904. }
  5905. static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
  5906. {
  5907. int r;
  5908. r = vcpu_load(vcpu);
  5909. BUG_ON(r);
  5910. kvm_mmu_unload(vcpu);
  5911. vcpu_put(vcpu);
  5912. }
  5913. static void kvm_free_vcpus(struct kvm *kvm)
  5914. {
  5915. unsigned int i;
  5916. struct kvm_vcpu *vcpu;
  5917. /*
  5918. * Unpin any mmu pages first.
  5919. */
  5920. kvm_for_each_vcpu(i, vcpu, kvm) {
  5921. kvm_clear_async_pf_completion_queue(vcpu);
  5922. kvm_unload_vcpu_mmu(vcpu);
  5923. }
  5924. kvm_for_each_vcpu(i, vcpu, kvm)
  5925. kvm_arch_vcpu_free(vcpu);
  5926. mutex_lock(&kvm->lock);
  5927. for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
  5928. kvm->vcpus[i] = NULL;
  5929. atomic_set(&kvm->online_vcpus, 0);
  5930. mutex_unlock(&kvm->lock);
  5931. }
  5932. void kvm_arch_sync_events(struct kvm *kvm)
  5933. {
  5934. kvm_free_all_assigned_devices(kvm);
  5935. kvm_free_pit(kvm);
  5936. }
  5937. void kvm_arch_destroy_vm(struct kvm *kvm)
  5938. {
  5939. if (current->mm == kvm->mm) {
  5940. /*
  5941. * Free memory regions allocated on behalf of userspace,
  5942. * unless the the memory map has changed due to process exit
  5943. * or fd copying.
  5944. */
  5945. struct kvm_userspace_memory_region mem;
  5946. memset(&mem, 0, sizeof(mem));
  5947. mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
  5948. kvm_set_memory_region(kvm, &mem);
  5949. mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
  5950. kvm_set_memory_region(kvm, &mem);
  5951. mem.slot = TSS_PRIVATE_MEMSLOT;
  5952. kvm_set_memory_region(kvm, &mem);
  5953. }
  5954. kvm_iommu_unmap_guest(kvm);
  5955. kfree(kvm->arch.vpic);
  5956. kfree(kvm->arch.vioapic);
  5957. kvm_free_vcpus(kvm);
  5958. if (kvm->arch.apic_access_page)
  5959. put_page(kvm->arch.apic_access_page);
  5960. if (kvm->arch.ept_identity_pagetable)
  5961. put_page(kvm->arch.ept_identity_pagetable);
  5962. kfree(rcu_dereference_check(kvm->arch.apic_map, 1));
  5963. }
  5964. void kvm_arch_free_memslot(struct kvm_memory_slot *free,
  5965. struct kvm_memory_slot *dont)
  5966. {
  5967. int i;
  5968. for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
  5969. if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
  5970. kvm_kvfree(free->arch.rmap[i]);
  5971. free->arch.rmap[i] = NULL;
  5972. }
  5973. if (i == 0)
  5974. continue;
  5975. if (!dont || free->arch.lpage_info[i - 1] !=
  5976. dont->arch.lpage_info[i - 1]) {
  5977. kvm_kvfree(free->arch.lpage_info[i - 1]);
  5978. free->arch.lpage_info[i - 1] = NULL;
  5979. }
  5980. }
  5981. }
  5982. int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
  5983. {
  5984. int i;
  5985. for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
  5986. unsigned long ugfn;
  5987. int lpages;
  5988. int level = i + 1;
  5989. lpages = gfn_to_index(slot->base_gfn + npages - 1,
  5990. slot->base_gfn, level) + 1;
  5991. slot->arch.rmap[i] =
  5992. kvm_kvzalloc(lpages * sizeof(*slot->arch.rmap[i]));
  5993. if (!slot->arch.rmap[i])
  5994. goto out_free;
  5995. if (i == 0)
  5996. continue;
  5997. slot->arch.lpage_info[i - 1] = kvm_kvzalloc(lpages *
  5998. sizeof(*slot->arch.lpage_info[i - 1]));
  5999. if (!slot->arch.lpage_info[i - 1])
  6000. goto out_free;
  6001. if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
  6002. slot->arch.lpage_info[i - 1][0].write_count = 1;
  6003. if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
  6004. slot->arch.lpage_info[i - 1][lpages - 1].write_count = 1;
  6005. ugfn = slot->userspace_addr >> PAGE_SHIFT;
  6006. /*
  6007. * If the gfn and userspace address are not aligned wrt each
  6008. * other, or if explicitly asked to, disable large page
  6009. * support for this slot
  6010. */
  6011. if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
  6012. !kvm_largepages_enabled()) {
  6013. unsigned long j;
  6014. for (j = 0; j < lpages; ++j)
  6015. slot->arch.lpage_info[i - 1][j].write_count = 1;
  6016. }
  6017. }
  6018. return 0;
  6019. out_free:
  6020. for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
  6021. kvm_kvfree(slot->arch.rmap[i]);
  6022. slot->arch.rmap[i] = NULL;
  6023. if (i == 0)
  6024. continue;
  6025. kvm_kvfree(slot->arch.lpage_info[i - 1]);
  6026. slot->arch.lpage_info[i - 1] = NULL;
  6027. }
  6028. return -ENOMEM;
  6029. }
  6030. int kvm_arch_prepare_memory_region(struct kvm *kvm,
  6031. struct kvm_memory_slot *memslot,
  6032. struct kvm_userspace_memory_region *mem,
  6033. enum kvm_mr_change change)
  6034. {
  6035. /*
  6036. * Only private memory slots need to be mapped here since
  6037. * KVM_SET_MEMORY_REGION ioctl is no longer supported.
  6038. */
  6039. if ((memslot->id >= KVM_USER_MEM_SLOTS) && (change == KVM_MR_CREATE)) {
  6040. unsigned long userspace_addr;
  6041. /*
  6042. * MAP_SHARED to prevent internal slot pages from being moved
  6043. * by fork()/COW.
  6044. */
  6045. userspace_addr = vm_mmap(NULL, 0, memslot->npages * PAGE_SIZE,
  6046. PROT_READ | PROT_WRITE,
  6047. MAP_SHARED | MAP_ANONYMOUS, 0);
  6048. if (IS_ERR((void *)userspace_addr))
  6049. return PTR_ERR((void *)userspace_addr);
  6050. memslot->userspace_addr = userspace_addr;
  6051. }
  6052. return 0;
  6053. }
  6054. void kvm_arch_commit_memory_region(struct kvm *kvm,
  6055. struct kvm_userspace_memory_region *mem,
  6056. const struct kvm_memory_slot *old,
  6057. enum kvm_mr_change change)
  6058. {
  6059. int nr_mmu_pages = 0;
  6060. if ((mem->slot >= KVM_USER_MEM_SLOTS) && (change == KVM_MR_DELETE)) {
  6061. int ret;
  6062. ret = vm_munmap(old->userspace_addr,
  6063. old->npages * PAGE_SIZE);
  6064. if (ret < 0)
  6065. printk(KERN_WARNING
  6066. "kvm_vm_ioctl_set_memory_region: "
  6067. "failed to munmap memory\n");
  6068. }
  6069. if (!kvm->arch.n_requested_mmu_pages)
  6070. nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
  6071. if (nr_mmu_pages)
  6072. kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
  6073. /*
  6074. * Write protect all pages for dirty logging.
  6075. * Existing largepage mappings are destroyed here and new ones will
  6076. * not be created until the end of the logging.
  6077. */
  6078. if ((change != KVM_MR_DELETE) && (mem->flags & KVM_MEM_LOG_DIRTY_PAGES))
  6079. kvm_mmu_slot_remove_write_access(kvm, mem->slot);
  6080. /*
  6081. * If memory slot is created, or moved, we need to clear all
  6082. * mmio sptes.
  6083. */
  6084. kvm_mmu_invalidate_mmio_sptes(kvm);
  6085. }
  6086. void kvm_arch_flush_shadow_all(struct kvm *kvm)
  6087. {
  6088. kvm_mmu_invalidate_zap_all_pages(kvm);
  6089. }
  6090. void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
  6091. struct kvm_memory_slot *slot)
  6092. {
  6093. kvm_mmu_invalidate_zap_all_pages(kvm);
  6094. }
  6095. int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
  6096. {
  6097. return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
  6098. !vcpu->arch.apf.halted)
  6099. || !list_empty_careful(&vcpu->async_pf.done)
  6100. || kvm_apic_has_events(vcpu)
  6101. || atomic_read(&vcpu->arch.nmi_queued) ||
  6102. (kvm_arch_interrupt_allowed(vcpu) &&
  6103. kvm_cpu_has_interrupt(vcpu));
  6104. }
  6105. int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
  6106. {
  6107. return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
  6108. }
  6109. int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
  6110. {
  6111. return kvm_x86_ops->interrupt_allowed(vcpu);
  6112. }
  6113. bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
  6114. {
  6115. unsigned long current_rip = kvm_rip_read(vcpu) +
  6116. get_segment_base(vcpu, VCPU_SREG_CS);
  6117. return current_rip == linear_rip;
  6118. }
  6119. EXPORT_SYMBOL_GPL(kvm_is_linear_rip);
  6120. unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
  6121. {
  6122. unsigned long rflags;
  6123. rflags = kvm_x86_ops->get_rflags(vcpu);
  6124. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
  6125. rflags &= ~X86_EFLAGS_TF;
  6126. return rflags;
  6127. }
  6128. EXPORT_SYMBOL_GPL(kvm_get_rflags);
  6129. void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  6130. {
  6131. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
  6132. kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
  6133. rflags |= X86_EFLAGS_TF;
  6134. kvm_x86_ops->set_rflags(vcpu, rflags);
  6135. kvm_make_request(KVM_REQ_EVENT, vcpu);
  6136. }
  6137. EXPORT_SYMBOL_GPL(kvm_set_rflags);
  6138. void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
  6139. {
  6140. int r;
  6141. if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) ||
  6142. is_error_page(work->page))
  6143. return;
  6144. r = kvm_mmu_reload(vcpu);
  6145. if (unlikely(r))
  6146. return;
  6147. if (!vcpu->arch.mmu.direct_map &&
  6148. work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu))
  6149. return;
  6150. vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true);
  6151. }
  6152. static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
  6153. {
  6154. return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
  6155. }
  6156. static inline u32 kvm_async_pf_next_probe(u32 key)
  6157. {
  6158. return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
  6159. }
  6160. static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
  6161. {
  6162. u32 key = kvm_async_pf_hash_fn(gfn);
  6163. while (vcpu->arch.apf.gfns[key] != ~0)
  6164. key = kvm_async_pf_next_probe(key);
  6165. vcpu->arch.apf.gfns[key] = gfn;
  6166. }
  6167. static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
  6168. {
  6169. int i;
  6170. u32 key = kvm_async_pf_hash_fn(gfn);
  6171. for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
  6172. (vcpu->arch.apf.gfns[key] != gfn &&
  6173. vcpu->arch.apf.gfns[key] != ~0); i++)
  6174. key = kvm_async_pf_next_probe(key);
  6175. return key;
  6176. }
  6177. bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
  6178. {
  6179. return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
  6180. }
  6181. static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
  6182. {
  6183. u32 i, j, k;
  6184. i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
  6185. while (true) {
  6186. vcpu->arch.apf.gfns[i] = ~0;
  6187. do {
  6188. j = kvm_async_pf_next_probe(j);
  6189. if (vcpu->arch.apf.gfns[j] == ~0)
  6190. return;
  6191. k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
  6192. /*
  6193. * k lies cyclically in ]i,j]
  6194. * | i.k.j |
  6195. * |....j i.k.| or |.k..j i...|
  6196. */
  6197. } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
  6198. vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
  6199. i = j;
  6200. }
  6201. }
  6202. static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
  6203. {
  6204. return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
  6205. sizeof(val));
  6206. }
  6207. void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
  6208. struct kvm_async_pf *work)
  6209. {
  6210. struct x86_exception fault;
  6211. trace_kvm_async_pf_not_present(work->arch.token, work->gva);
  6212. kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
  6213. if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
  6214. (vcpu->arch.apf.send_user_only &&
  6215. kvm_x86_ops->get_cpl(vcpu) == 0))
  6216. kvm_make_request(KVM_REQ_APF_HALT, vcpu);
  6217. else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
  6218. fault.vector = PF_VECTOR;
  6219. fault.error_code_valid = true;
  6220. fault.error_code = 0;
  6221. fault.nested_page_fault = false;
  6222. fault.address = work->arch.token;
  6223. kvm_inject_page_fault(vcpu, &fault);
  6224. }
  6225. }
  6226. void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
  6227. struct kvm_async_pf *work)
  6228. {
  6229. struct x86_exception fault;
  6230. trace_kvm_async_pf_ready(work->arch.token, work->gva);
  6231. if (is_error_page(work->page))
  6232. work->arch.token = ~0; /* broadcast wakeup */
  6233. else
  6234. kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
  6235. if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) &&
  6236. !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
  6237. fault.vector = PF_VECTOR;
  6238. fault.error_code_valid = true;
  6239. fault.error_code = 0;
  6240. fault.nested_page_fault = false;
  6241. fault.address = work->arch.token;
  6242. kvm_inject_page_fault(vcpu, &fault);
  6243. }
  6244. vcpu->arch.apf.halted = false;
  6245. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  6246. }
  6247. bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
  6248. {
  6249. if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
  6250. return true;
  6251. else
  6252. return !kvm_event_needs_reinjection(vcpu) &&
  6253. kvm_x86_ops->interrupt_allowed(vcpu);
  6254. }
  6255. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
  6256. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
  6257. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
  6258. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
  6259. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
  6260. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
  6261. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
  6262. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
  6263. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
  6264. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
  6265. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
  6266. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
  6267. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);