sched.c 222 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240
  1. /*
  2. * kernel/sched.c
  3. *
  4. * Kernel scheduler and related syscalls
  5. *
  6. * Copyright (C) 1991-2002 Linus Torvalds
  7. *
  8. * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and
  9. * make semaphores SMP safe
  10. * 1998-11-19 Implemented schedule_timeout() and related stuff
  11. * by Andrea Arcangeli
  12. * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:
  13. * hybrid priority-list and round-robin design with
  14. * an array-switch method of distributing timeslices
  15. * and per-CPU runqueues. Cleanups and useful suggestions
  16. * by Davide Libenzi, preemptible kernel bits by Robert Love.
  17. * 2003-09-03 Interactivity tuning by Con Kolivas.
  18. * 2004-04-02 Scheduler domains code by Nick Piggin
  19. * 2007-04-15 Work begun on replacing all interactivity tuning with a
  20. * fair scheduling design by Con Kolivas.
  21. * 2007-05-05 Load balancing (smp-nice) and other improvements
  22. * by Peter Williams
  23. * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith
  24. * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri
  25. * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins,
  26. * Thomas Gleixner, Mike Kravetz
  27. */
  28. #include <linux/mm.h>
  29. #include <linux/module.h>
  30. #include <linux/nmi.h>
  31. #include <linux/init.h>
  32. #include <linux/uaccess.h>
  33. #include <linux/highmem.h>
  34. #include <linux/smp_lock.h>
  35. #include <asm/mmu_context.h>
  36. #include <linux/interrupt.h>
  37. #include <linux/capability.h>
  38. #include <linux/completion.h>
  39. #include <linux/kernel_stat.h>
  40. #include <linux/debug_locks.h>
  41. #include <linux/perf_event.h>
  42. #include <linux/security.h>
  43. #include <linux/notifier.h>
  44. #include <linux/profile.h>
  45. #include <linux/freezer.h>
  46. #include <linux/vmalloc.h>
  47. #include <linux/blkdev.h>
  48. #include <linux/delay.h>
  49. #include <linux/pid_namespace.h>
  50. #include <linux/smp.h>
  51. #include <linux/threads.h>
  52. #include <linux/timer.h>
  53. #include <linux/rcupdate.h>
  54. #include <linux/cpu.h>
  55. #include <linux/cpuset.h>
  56. #include <linux/percpu.h>
  57. #include <linux/proc_fs.h>
  58. #include <linux/seq_file.h>
  59. #include <linux/stop_machine.h>
  60. #include <linux/sysctl.h>
  61. #include <linux/syscalls.h>
  62. #include <linux/times.h>
  63. #include <linux/tsacct_kern.h>
  64. #include <linux/kprobes.h>
  65. #include <linux/delayacct.h>
  66. #include <linux/unistd.h>
  67. #include <linux/pagemap.h>
  68. #include <linux/hrtimer.h>
  69. #include <linux/tick.h>
  70. #include <linux/debugfs.h>
  71. #include <linux/ctype.h>
  72. #include <linux/ftrace.h>
  73. #include <linux/slab.h>
  74. #include <asm/tlb.h>
  75. #include <asm/irq_regs.h>
  76. #include <asm/mutex.h>
  77. #include "sched_cpupri.h"
  78. #include "workqueue_sched.h"
  79. #include "sched_autogroup.h"
  80. #define CREATE_TRACE_POINTS
  81. #include <trace/events/sched.h>
  82. /*
  83. * Convert user-nice values [ -20 ... 0 ... 19 ]
  84. * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
  85. * and back.
  86. */
  87. #define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20)
  88. #define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20)
  89. #define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio)
  90. /*
  91. * 'User priority' is the nice value converted to something we
  92. * can work with better when scaling various scheduler parameters,
  93. * it's a [ 0 ... 39 ] range.
  94. */
  95. #define USER_PRIO(p) ((p)-MAX_RT_PRIO)
  96. #define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)
  97. #define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
  98. /*
  99. * Helpers for converting nanosecond timing to jiffy resolution
  100. */
  101. #define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
  102. #define NICE_0_LOAD SCHED_LOAD_SCALE
  103. #define NICE_0_SHIFT SCHED_LOAD_SHIFT
  104. /*
  105. * These are the 'tuning knobs' of the scheduler:
  106. *
  107. * default timeslice is 100 msecs (used only for SCHED_RR tasks).
  108. * Timeslices get refilled after they expire.
  109. */
  110. #define DEF_TIMESLICE (100 * HZ / 1000)
  111. /*
  112. * single value that denotes runtime == period, ie unlimited time.
  113. */
  114. #define RUNTIME_INF ((u64)~0ULL)
  115. static inline int rt_policy(int policy)
  116. {
  117. if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
  118. return 1;
  119. return 0;
  120. }
  121. static inline int task_has_rt_policy(struct task_struct *p)
  122. {
  123. return rt_policy(p->policy);
  124. }
  125. /*
  126. * This is the priority-queue data structure of the RT scheduling class:
  127. */
  128. struct rt_prio_array {
  129. DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
  130. struct list_head queue[MAX_RT_PRIO];
  131. };
  132. struct rt_bandwidth {
  133. /* nests inside the rq lock: */
  134. raw_spinlock_t rt_runtime_lock;
  135. ktime_t rt_period;
  136. u64 rt_runtime;
  137. struct hrtimer rt_period_timer;
  138. };
  139. static struct rt_bandwidth def_rt_bandwidth;
  140. static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
  141. static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
  142. {
  143. struct rt_bandwidth *rt_b =
  144. container_of(timer, struct rt_bandwidth, rt_period_timer);
  145. ktime_t now;
  146. int overrun;
  147. int idle = 0;
  148. for (;;) {
  149. now = hrtimer_cb_get_time(timer);
  150. overrun = hrtimer_forward(timer, now, rt_b->rt_period);
  151. if (!overrun)
  152. break;
  153. idle = do_sched_rt_period_timer(rt_b, overrun);
  154. }
  155. return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
  156. }
  157. static
  158. void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
  159. {
  160. rt_b->rt_period = ns_to_ktime(period);
  161. rt_b->rt_runtime = runtime;
  162. raw_spin_lock_init(&rt_b->rt_runtime_lock);
  163. hrtimer_init(&rt_b->rt_period_timer,
  164. CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  165. rt_b->rt_period_timer.function = sched_rt_period_timer;
  166. }
  167. static inline int rt_bandwidth_enabled(void)
  168. {
  169. return sysctl_sched_rt_runtime >= 0;
  170. }
  171. static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
  172. {
  173. ktime_t now;
  174. if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
  175. return;
  176. if (hrtimer_active(&rt_b->rt_period_timer))
  177. return;
  178. raw_spin_lock(&rt_b->rt_runtime_lock);
  179. for (;;) {
  180. unsigned long delta;
  181. ktime_t soft, hard;
  182. if (hrtimer_active(&rt_b->rt_period_timer))
  183. break;
  184. now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
  185. hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
  186. soft = hrtimer_get_softexpires(&rt_b->rt_period_timer);
  187. hard = hrtimer_get_expires(&rt_b->rt_period_timer);
  188. delta = ktime_to_ns(ktime_sub(hard, soft));
  189. __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,
  190. HRTIMER_MODE_ABS_PINNED, 0);
  191. }
  192. raw_spin_unlock(&rt_b->rt_runtime_lock);
  193. }
  194. #ifdef CONFIG_RT_GROUP_SCHED
  195. static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
  196. {
  197. hrtimer_cancel(&rt_b->rt_period_timer);
  198. }
  199. #endif
  200. /*
  201. * sched_domains_mutex serializes calls to arch_init_sched_domains,
  202. * detach_destroy_domains and partition_sched_domains.
  203. */
  204. static DEFINE_MUTEX(sched_domains_mutex);
  205. #ifdef CONFIG_CGROUP_SCHED
  206. #include <linux/cgroup.h>
  207. struct cfs_rq;
  208. static LIST_HEAD(task_groups);
  209. /* task group related information */
  210. struct task_group {
  211. struct cgroup_subsys_state css;
  212. #ifdef CONFIG_FAIR_GROUP_SCHED
  213. /* schedulable entities of this group on each cpu */
  214. struct sched_entity **se;
  215. /* runqueue "owned" by this group on each cpu */
  216. struct cfs_rq **cfs_rq;
  217. unsigned long shares;
  218. atomic_t load_weight;
  219. #endif
  220. #ifdef CONFIG_RT_GROUP_SCHED
  221. struct sched_rt_entity **rt_se;
  222. struct rt_rq **rt_rq;
  223. struct rt_bandwidth rt_bandwidth;
  224. #endif
  225. struct rcu_head rcu;
  226. struct list_head list;
  227. struct task_group *parent;
  228. struct list_head siblings;
  229. struct list_head children;
  230. #ifdef CONFIG_SCHED_AUTOGROUP
  231. struct autogroup *autogroup;
  232. #endif
  233. };
  234. /* task_group_lock serializes the addition/removal of task groups */
  235. static DEFINE_SPINLOCK(task_group_lock);
  236. #ifdef CONFIG_FAIR_GROUP_SCHED
  237. # define ROOT_TASK_GROUP_LOAD NICE_0_LOAD
  238. /*
  239. * A weight of 0 or 1 can cause arithmetics problems.
  240. * A weight of a cfs_rq is the sum of weights of which entities
  241. * are queued on this cfs_rq, so a weight of a entity should not be
  242. * too large, so as the shares value of a task group.
  243. * (The default weight is 1024 - so there's no practical
  244. * limitation from this.)
  245. */
  246. #define MIN_SHARES 2
  247. #define MAX_SHARES (1UL << 18)
  248. static int root_task_group_load = ROOT_TASK_GROUP_LOAD;
  249. #endif
  250. /* Default task group.
  251. * Every task in system belong to this group at bootup.
  252. */
  253. struct task_group root_task_group;
  254. #endif /* CONFIG_CGROUP_SCHED */
  255. /* CFS-related fields in a runqueue */
  256. struct cfs_rq {
  257. struct load_weight load;
  258. unsigned long nr_running;
  259. u64 exec_clock;
  260. u64 min_vruntime;
  261. struct rb_root tasks_timeline;
  262. struct rb_node *rb_leftmost;
  263. struct list_head tasks;
  264. struct list_head *balance_iterator;
  265. /*
  266. * 'curr' points to currently running entity on this cfs_rq.
  267. * It is set to NULL otherwise (i.e when none are currently running).
  268. */
  269. struct sched_entity *curr, *next, *last;
  270. unsigned int nr_spread_over;
  271. #ifdef CONFIG_FAIR_GROUP_SCHED
  272. struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */
  273. /*
  274. * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
  275. * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
  276. * (like users, containers etc.)
  277. *
  278. * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
  279. * list is used during load balance.
  280. */
  281. int on_list;
  282. struct list_head leaf_cfs_rq_list;
  283. struct task_group *tg; /* group that "owns" this runqueue */
  284. #ifdef CONFIG_SMP
  285. /*
  286. * the part of load.weight contributed by tasks
  287. */
  288. unsigned long task_weight;
  289. /*
  290. * h_load = weight * f(tg)
  291. *
  292. * Where f(tg) is the recursive weight fraction assigned to
  293. * this group.
  294. */
  295. unsigned long h_load;
  296. /*
  297. * Maintaining per-cpu shares distribution for group scheduling
  298. *
  299. * load_stamp is the last time we updated the load average
  300. * load_last is the last time we updated the load average and saw load
  301. * load_unacc_exec_time is currently unaccounted execution time
  302. */
  303. u64 load_avg;
  304. u64 load_period;
  305. u64 load_stamp, load_last, load_unacc_exec_time;
  306. unsigned long load_contribution;
  307. #endif
  308. #endif
  309. };
  310. /* Real-Time classes' related field in a runqueue: */
  311. struct rt_rq {
  312. struct rt_prio_array active;
  313. unsigned long rt_nr_running;
  314. #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
  315. struct {
  316. int curr; /* highest queued rt task prio */
  317. #ifdef CONFIG_SMP
  318. int next; /* next highest */
  319. #endif
  320. } highest_prio;
  321. #endif
  322. #ifdef CONFIG_SMP
  323. unsigned long rt_nr_migratory;
  324. unsigned long rt_nr_total;
  325. int overloaded;
  326. struct plist_head pushable_tasks;
  327. #endif
  328. int rt_throttled;
  329. u64 rt_time;
  330. u64 rt_runtime;
  331. /* Nests inside the rq lock: */
  332. raw_spinlock_t rt_runtime_lock;
  333. #ifdef CONFIG_RT_GROUP_SCHED
  334. unsigned long rt_nr_boosted;
  335. struct rq *rq;
  336. struct list_head leaf_rt_rq_list;
  337. struct task_group *tg;
  338. #endif
  339. };
  340. #ifdef CONFIG_SMP
  341. /*
  342. * We add the notion of a root-domain which will be used to define per-domain
  343. * variables. Each exclusive cpuset essentially defines an island domain by
  344. * fully partitioning the member cpus from any other cpuset. Whenever a new
  345. * exclusive cpuset is created, we also create and attach a new root-domain
  346. * object.
  347. *
  348. */
  349. struct root_domain {
  350. atomic_t refcount;
  351. cpumask_var_t span;
  352. cpumask_var_t online;
  353. /*
  354. * The "RT overload" flag: it gets set if a CPU has more than
  355. * one runnable RT task.
  356. */
  357. cpumask_var_t rto_mask;
  358. atomic_t rto_count;
  359. struct cpupri cpupri;
  360. };
  361. /*
  362. * By default the system creates a single root-domain with all cpus as
  363. * members (mimicking the global state we have today).
  364. */
  365. static struct root_domain def_root_domain;
  366. #endif /* CONFIG_SMP */
  367. /*
  368. * This is the main, per-CPU runqueue data structure.
  369. *
  370. * Locking rule: those places that want to lock multiple runqueues
  371. * (such as the load balancing or the thread migration code), lock
  372. * acquire operations must be ordered by ascending &runqueue.
  373. */
  374. struct rq {
  375. /* runqueue lock: */
  376. raw_spinlock_t lock;
  377. /*
  378. * nr_running and cpu_load should be in the same cacheline because
  379. * remote CPUs use both these fields when doing load calculation.
  380. */
  381. unsigned long nr_running;
  382. #define CPU_LOAD_IDX_MAX 5
  383. unsigned long cpu_load[CPU_LOAD_IDX_MAX];
  384. unsigned long last_load_update_tick;
  385. #ifdef CONFIG_NO_HZ
  386. u64 nohz_stamp;
  387. unsigned char nohz_balance_kick;
  388. #endif
  389. unsigned int skip_clock_update;
  390. /* capture load from *all* tasks on this cpu: */
  391. struct load_weight load;
  392. unsigned long nr_load_updates;
  393. u64 nr_switches;
  394. struct cfs_rq cfs;
  395. struct rt_rq rt;
  396. #ifdef CONFIG_FAIR_GROUP_SCHED
  397. /* list of leaf cfs_rq on this cpu: */
  398. struct list_head leaf_cfs_rq_list;
  399. #endif
  400. #ifdef CONFIG_RT_GROUP_SCHED
  401. struct list_head leaf_rt_rq_list;
  402. #endif
  403. /*
  404. * This is part of a global counter where only the total sum
  405. * over all CPUs matters. A task can increase this counter on
  406. * one CPU and if it got migrated afterwards it may decrease
  407. * it on another CPU. Always updated under the runqueue lock:
  408. */
  409. unsigned long nr_uninterruptible;
  410. struct task_struct *curr, *idle, *stop;
  411. unsigned long next_balance;
  412. struct mm_struct *prev_mm;
  413. u64 clock;
  414. u64 clock_task;
  415. atomic_t nr_iowait;
  416. #ifdef CONFIG_SMP
  417. struct root_domain *rd;
  418. struct sched_domain *sd;
  419. unsigned long cpu_power;
  420. unsigned char idle_at_tick;
  421. /* For active balancing */
  422. int post_schedule;
  423. int active_balance;
  424. int push_cpu;
  425. struct cpu_stop_work active_balance_work;
  426. /* cpu of this runqueue: */
  427. int cpu;
  428. int online;
  429. unsigned long avg_load_per_task;
  430. u64 rt_avg;
  431. u64 age_stamp;
  432. u64 idle_stamp;
  433. u64 avg_idle;
  434. #endif
  435. #ifdef CONFIG_IRQ_TIME_ACCOUNTING
  436. u64 prev_irq_time;
  437. #endif
  438. /* calc_load related fields */
  439. unsigned long calc_load_update;
  440. long calc_load_active;
  441. #ifdef CONFIG_SCHED_HRTICK
  442. #ifdef CONFIG_SMP
  443. int hrtick_csd_pending;
  444. struct call_single_data hrtick_csd;
  445. #endif
  446. struct hrtimer hrtick_timer;
  447. #endif
  448. #ifdef CONFIG_SCHEDSTATS
  449. /* latency stats */
  450. struct sched_info rq_sched_info;
  451. unsigned long long rq_cpu_time;
  452. /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
  453. /* sys_sched_yield() stats */
  454. unsigned int yld_count;
  455. /* schedule() stats */
  456. unsigned int sched_switch;
  457. unsigned int sched_count;
  458. unsigned int sched_goidle;
  459. /* try_to_wake_up() stats */
  460. unsigned int ttwu_count;
  461. unsigned int ttwu_local;
  462. /* BKL stats */
  463. unsigned int bkl_count;
  464. #endif
  465. };
  466. static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
  467. static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
  468. static inline int cpu_of(struct rq *rq)
  469. {
  470. #ifdef CONFIG_SMP
  471. return rq->cpu;
  472. #else
  473. return 0;
  474. #endif
  475. }
  476. #define rcu_dereference_check_sched_domain(p) \
  477. rcu_dereference_check((p), \
  478. rcu_read_lock_sched_held() || \
  479. lockdep_is_held(&sched_domains_mutex))
  480. /*
  481. * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
  482. * See detach_destroy_domains: synchronize_sched for details.
  483. *
  484. * The domain tree of any CPU may only be accessed from within
  485. * preempt-disabled sections.
  486. */
  487. #define for_each_domain(cpu, __sd) \
  488. for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
  489. #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
  490. #define this_rq() (&__get_cpu_var(runqueues))
  491. #define task_rq(p) cpu_rq(task_cpu(p))
  492. #define cpu_curr(cpu) (cpu_rq(cpu)->curr)
  493. #define raw_rq() (&__raw_get_cpu_var(runqueues))
  494. #ifdef CONFIG_CGROUP_SCHED
  495. /*
  496. * Return the group to which this tasks belongs.
  497. *
  498. * We use task_subsys_state_check() and extend the RCU verification
  499. * with lockdep_is_held(&task_rq(p)->lock) because cpu_cgroup_attach()
  500. * holds that lock for each task it moves into the cgroup. Therefore
  501. * by holding that lock, we pin the task to the current cgroup.
  502. */
  503. static inline struct task_group *task_group(struct task_struct *p)
  504. {
  505. struct task_group *tg;
  506. struct cgroup_subsys_state *css;
  507. css = task_subsys_state_check(p, cpu_cgroup_subsys_id,
  508. lockdep_is_held(&task_rq(p)->lock));
  509. tg = container_of(css, struct task_group, css);
  510. return autogroup_task_group(p, tg);
  511. }
  512. /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
  513. static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
  514. {
  515. #ifdef CONFIG_FAIR_GROUP_SCHED
  516. p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
  517. p->se.parent = task_group(p)->se[cpu];
  518. #endif
  519. #ifdef CONFIG_RT_GROUP_SCHED
  520. p->rt.rt_rq = task_group(p)->rt_rq[cpu];
  521. p->rt.parent = task_group(p)->rt_se[cpu];
  522. #endif
  523. }
  524. #else /* CONFIG_CGROUP_SCHED */
  525. static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
  526. static inline struct task_group *task_group(struct task_struct *p)
  527. {
  528. return NULL;
  529. }
  530. #endif /* CONFIG_CGROUP_SCHED */
  531. static void update_rq_clock_task(struct rq *rq, s64 delta);
  532. static void update_rq_clock(struct rq *rq)
  533. {
  534. s64 delta;
  535. if (rq->skip_clock_update)
  536. return;
  537. delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
  538. rq->clock += delta;
  539. update_rq_clock_task(rq, delta);
  540. }
  541. /*
  542. * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
  543. */
  544. #ifdef CONFIG_SCHED_DEBUG
  545. # define const_debug __read_mostly
  546. #else
  547. # define const_debug static const
  548. #endif
  549. /**
  550. * runqueue_is_locked
  551. * @cpu: the processor in question.
  552. *
  553. * Returns true if the current cpu runqueue is locked.
  554. * This interface allows printk to be called with the runqueue lock
  555. * held and know whether or not it is OK to wake up the klogd.
  556. */
  557. int runqueue_is_locked(int cpu)
  558. {
  559. return raw_spin_is_locked(&cpu_rq(cpu)->lock);
  560. }
  561. /*
  562. * Debugging: various feature bits
  563. */
  564. #define SCHED_FEAT(name, enabled) \
  565. __SCHED_FEAT_##name ,
  566. enum {
  567. #include "sched_features.h"
  568. };
  569. #undef SCHED_FEAT
  570. #define SCHED_FEAT(name, enabled) \
  571. (1UL << __SCHED_FEAT_##name) * enabled |
  572. const_debug unsigned int sysctl_sched_features =
  573. #include "sched_features.h"
  574. 0;
  575. #undef SCHED_FEAT
  576. #ifdef CONFIG_SCHED_DEBUG
  577. #define SCHED_FEAT(name, enabled) \
  578. #name ,
  579. static __read_mostly char *sched_feat_names[] = {
  580. #include "sched_features.h"
  581. NULL
  582. };
  583. #undef SCHED_FEAT
  584. static int sched_feat_show(struct seq_file *m, void *v)
  585. {
  586. int i;
  587. for (i = 0; sched_feat_names[i]; i++) {
  588. if (!(sysctl_sched_features & (1UL << i)))
  589. seq_puts(m, "NO_");
  590. seq_printf(m, "%s ", sched_feat_names[i]);
  591. }
  592. seq_puts(m, "\n");
  593. return 0;
  594. }
  595. static ssize_t
  596. sched_feat_write(struct file *filp, const char __user *ubuf,
  597. size_t cnt, loff_t *ppos)
  598. {
  599. char buf[64];
  600. char *cmp;
  601. int neg = 0;
  602. int i;
  603. if (cnt > 63)
  604. cnt = 63;
  605. if (copy_from_user(&buf, ubuf, cnt))
  606. return -EFAULT;
  607. buf[cnt] = 0;
  608. cmp = strstrip(buf);
  609. if (strncmp(cmp, "NO_", 3) == 0) {
  610. neg = 1;
  611. cmp += 3;
  612. }
  613. for (i = 0; sched_feat_names[i]; i++) {
  614. if (strcmp(cmp, sched_feat_names[i]) == 0) {
  615. if (neg)
  616. sysctl_sched_features &= ~(1UL << i);
  617. else
  618. sysctl_sched_features |= (1UL << i);
  619. break;
  620. }
  621. }
  622. if (!sched_feat_names[i])
  623. return -EINVAL;
  624. *ppos += cnt;
  625. return cnt;
  626. }
  627. static int sched_feat_open(struct inode *inode, struct file *filp)
  628. {
  629. return single_open(filp, sched_feat_show, NULL);
  630. }
  631. static const struct file_operations sched_feat_fops = {
  632. .open = sched_feat_open,
  633. .write = sched_feat_write,
  634. .read = seq_read,
  635. .llseek = seq_lseek,
  636. .release = single_release,
  637. };
  638. static __init int sched_init_debug(void)
  639. {
  640. debugfs_create_file("sched_features", 0644, NULL, NULL,
  641. &sched_feat_fops);
  642. return 0;
  643. }
  644. late_initcall(sched_init_debug);
  645. #endif
  646. #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
  647. /*
  648. * Number of tasks to iterate in a single balance run.
  649. * Limited because this is done with IRQs disabled.
  650. */
  651. const_debug unsigned int sysctl_sched_nr_migrate = 32;
  652. /*
  653. * period over which we average the RT time consumption, measured
  654. * in ms.
  655. *
  656. * default: 1s
  657. */
  658. const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
  659. /*
  660. * period over which we measure -rt task cpu usage in us.
  661. * default: 1s
  662. */
  663. unsigned int sysctl_sched_rt_period = 1000000;
  664. static __read_mostly int scheduler_running;
  665. /*
  666. * part of the period that we allow rt tasks to run in us.
  667. * default: 0.95s
  668. */
  669. int sysctl_sched_rt_runtime = 950000;
  670. static inline u64 global_rt_period(void)
  671. {
  672. return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
  673. }
  674. static inline u64 global_rt_runtime(void)
  675. {
  676. if (sysctl_sched_rt_runtime < 0)
  677. return RUNTIME_INF;
  678. return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
  679. }
  680. #ifndef prepare_arch_switch
  681. # define prepare_arch_switch(next) do { } while (0)
  682. #endif
  683. #ifndef finish_arch_switch
  684. # define finish_arch_switch(prev) do { } while (0)
  685. #endif
  686. static inline int task_current(struct rq *rq, struct task_struct *p)
  687. {
  688. return rq->curr == p;
  689. }
  690. #ifndef __ARCH_WANT_UNLOCKED_CTXSW
  691. static inline int task_running(struct rq *rq, struct task_struct *p)
  692. {
  693. return task_current(rq, p);
  694. }
  695. static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
  696. {
  697. }
  698. static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
  699. {
  700. #ifdef CONFIG_DEBUG_SPINLOCK
  701. /* this is a valid case when another task releases the spinlock */
  702. rq->lock.owner = current;
  703. #endif
  704. /*
  705. * If we are tracking spinlock dependencies then we have to
  706. * fix up the runqueue lock - which gets 'carried over' from
  707. * prev into current:
  708. */
  709. spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
  710. raw_spin_unlock_irq(&rq->lock);
  711. }
  712. #else /* __ARCH_WANT_UNLOCKED_CTXSW */
  713. static inline int task_running(struct rq *rq, struct task_struct *p)
  714. {
  715. #ifdef CONFIG_SMP
  716. return p->oncpu;
  717. #else
  718. return task_current(rq, p);
  719. #endif
  720. }
  721. static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
  722. {
  723. #ifdef CONFIG_SMP
  724. /*
  725. * We can optimise this out completely for !SMP, because the
  726. * SMP rebalancing from interrupt is the only thing that cares
  727. * here.
  728. */
  729. next->oncpu = 1;
  730. #endif
  731. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  732. raw_spin_unlock_irq(&rq->lock);
  733. #else
  734. raw_spin_unlock(&rq->lock);
  735. #endif
  736. }
  737. static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
  738. {
  739. #ifdef CONFIG_SMP
  740. /*
  741. * After ->oncpu is cleared, the task can be moved to a different CPU.
  742. * We must ensure this doesn't happen until the switch is completely
  743. * finished.
  744. */
  745. smp_wmb();
  746. prev->oncpu = 0;
  747. #endif
  748. #ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  749. local_irq_enable();
  750. #endif
  751. }
  752. #endif /* __ARCH_WANT_UNLOCKED_CTXSW */
  753. /*
  754. * Check whether the task is waking, we use this to synchronize ->cpus_allowed
  755. * against ttwu().
  756. */
  757. static inline int task_is_waking(struct task_struct *p)
  758. {
  759. return unlikely(p->state == TASK_WAKING);
  760. }
  761. /*
  762. * __task_rq_lock - lock the runqueue a given task resides on.
  763. * Must be called interrupts disabled.
  764. */
  765. static inline struct rq *__task_rq_lock(struct task_struct *p)
  766. __acquires(rq->lock)
  767. {
  768. struct rq *rq;
  769. for (;;) {
  770. rq = task_rq(p);
  771. raw_spin_lock(&rq->lock);
  772. if (likely(rq == task_rq(p)))
  773. return rq;
  774. raw_spin_unlock(&rq->lock);
  775. }
  776. }
  777. /*
  778. * task_rq_lock - lock the runqueue a given task resides on and disable
  779. * interrupts. Note the ordering: we can safely lookup the task_rq without
  780. * explicitly disabling preemption.
  781. */
  782. static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
  783. __acquires(rq->lock)
  784. {
  785. struct rq *rq;
  786. for (;;) {
  787. local_irq_save(*flags);
  788. rq = task_rq(p);
  789. raw_spin_lock(&rq->lock);
  790. if (likely(rq == task_rq(p)))
  791. return rq;
  792. raw_spin_unlock_irqrestore(&rq->lock, *flags);
  793. }
  794. }
  795. static void __task_rq_unlock(struct rq *rq)
  796. __releases(rq->lock)
  797. {
  798. raw_spin_unlock(&rq->lock);
  799. }
  800. static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
  801. __releases(rq->lock)
  802. {
  803. raw_spin_unlock_irqrestore(&rq->lock, *flags);
  804. }
  805. /*
  806. * this_rq_lock - lock this runqueue and disable interrupts.
  807. */
  808. static struct rq *this_rq_lock(void)
  809. __acquires(rq->lock)
  810. {
  811. struct rq *rq;
  812. local_irq_disable();
  813. rq = this_rq();
  814. raw_spin_lock(&rq->lock);
  815. return rq;
  816. }
  817. #ifdef CONFIG_SCHED_HRTICK
  818. /*
  819. * Use HR-timers to deliver accurate preemption points.
  820. *
  821. * Its all a bit involved since we cannot program an hrt while holding the
  822. * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
  823. * reschedule event.
  824. *
  825. * When we get rescheduled we reprogram the hrtick_timer outside of the
  826. * rq->lock.
  827. */
  828. /*
  829. * Use hrtick when:
  830. * - enabled by features
  831. * - hrtimer is actually high res
  832. */
  833. static inline int hrtick_enabled(struct rq *rq)
  834. {
  835. if (!sched_feat(HRTICK))
  836. return 0;
  837. if (!cpu_active(cpu_of(rq)))
  838. return 0;
  839. return hrtimer_is_hres_active(&rq->hrtick_timer);
  840. }
  841. static void hrtick_clear(struct rq *rq)
  842. {
  843. if (hrtimer_active(&rq->hrtick_timer))
  844. hrtimer_cancel(&rq->hrtick_timer);
  845. }
  846. /*
  847. * High-resolution timer tick.
  848. * Runs from hardirq context with interrupts disabled.
  849. */
  850. static enum hrtimer_restart hrtick(struct hrtimer *timer)
  851. {
  852. struct rq *rq = container_of(timer, struct rq, hrtick_timer);
  853. WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
  854. raw_spin_lock(&rq->lock);
  855. update_rq_clock(rq);
  856. rq->curr->sched_class->task_tick(rq, rq->curr, 1);
  857. raw_spin_unlock(&rq->lock);
  858. return HRTIMER_NORESTART;
  859. }
  860. #ifdef CONFIG_SMP
  861. /*
  862. * called from hardirq (IPI) context
  863. */
  864. static void __hrtick_start(void *arg)
  865. {
  866. struct rq *rq = arg;
  867. raw_spin_lock(&rq->lock);
  868. hrtimer_restart(&rq->hrtick_timer);
  869. rq->hrtick_csd_pending = 0;
  870. raw_spin_unlock(&rq->lock);
  871. }
  872. /*
  873. * Called to set the hrtick timer state.
  874. *
  875. * called with rq->lock held and irqs disabled
  876. */
  877. static void hrtick_start(struct rq *rq, u64 delay)
  878. {
  879. struct hrtimer *timer = &rq->hrtick_timer;
  880. ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
  881. hrtimer_set_expires(timer, time);
  882. if (rq == this_rq()) {
  883. hrtimer_restart(timer);
  884. } else if (!rq->hrtick_csd_pending) {
  885. __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);
  886. rq->hrtick_csd_pending = 1;
  887. }
  888. }
  889. static int
  890. hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
  891. {
  892. int cpu = (int)(long)hcpu;
  893. switch (action) {
  894. case CPU_UP_CANCELED:
  895. case CPU_UP_CANCELED_FROZEN:
  896. case CPU_DOWN_PREPARE:
  897. case CPU_DOWN_PREPARE_FROZEN:
  898. case CPU_DEAD:
  899. case CPU_DEAD_FROZEN:
  900. hrtick_clear(cpu_rq(cpu));
  901. return NOTIFY_OK;
  902. }
  903. return NOTIFY_DONE;
  904. }
  905. static __init void init_hrtick(void)
  906. {
  907. hotcpu_notifier(hotplug_hrtick, 0);
  908. }
  909. #else
  910. /*
  911. * Called to set the hrtick timer state.
  912. *
  913. * called with rq->lock held and irqs disabled
  914. */
  915. static void hrtick_start(struct rq *rq, u64 delay)
  916. {
  917. __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
  918. HRTIMER_MODE_REL_PINNED, 0);
  919. }
  920. static inline void init_hrtick(void)
  921. {
  922. }
  923. #endif /* CONFIG_SMP */
  924. static void init_rq_hrtick(struct rq *rq)
  925. {
  926. #ifdef CONFIG_SMP
  927. rq->hrtick_csd_pending = 0;
  928. rq->hrtick_csd.flags = 0;
  929. rq->hrtick_csd.func = __hrtick_start;
  930. rq->hrtick_csd.info = rq;
  931. #endif
  932. hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  933. rq->hrtick_timer.function = hrtick;
  934. }
  935. #else /* CONFIG_SCHED_HRTICK */
  936. static inline void hrtick_clear(struct rq *rq)
  937. {
  938. }
  939. static inline void init_rq_hrtick(struct rq *rq)
  940. {
  941. }
  942. static inline void init_hrtick(void)
  943. {
  944. }
  945. #endif /* CONFIG_SCHED_HRTICK */
  946. /*
  947. * resched_task - mark a task 'to be rescheduled now'.
  948. *
  949. * On UP this means the setting of the need_resched flag, on SMP it
  950. * might also involve a cross-CPU call to trigger the scheduler on
  951. * the target CPU.
  952. */
  953. #ifdef CONFIG_SMP
  954. #ifndef tsk_is_polling
  955. #define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
  956. #endif
  957. static void resched_task(struct task_struct *p)
  958. {
  959. int cpu;
  960. assert_raw_spin_locked(&task_rq(p)->lock);
  961. if (test_tsk_need_resched(p))
  962. return;
  963. set_tsk_need_resched(p);
  964. cpu = task_cpu(p);
  965. if (cpu == smp_processor_id())
  966. return;
  967. /* NEED_RESCHED must be visible before we test polling */
  968. smp_mb();
  969. if (!tsk_is_polling(p))
  970. smp_send_reschedule(cpu);
  971. }
  972. static void resched_cpu(int cpu)
  973. {
  974. struct rq *rq = cpu_rq(cpu);
  975. unsigned long flags;
  976. if (!raw_spin_trylock_irqsave(&rq->lock, flags))
  977. return;
  978. resched_task(cpu_curr(cpu));
  979. raw_spin_unlock_irqrestore(&rq->lock, flags);
  980. }
  981. #ifdef CONFIG_NO_HZ
  982. /*
  983. * In the semi idle case, use the nearest busy cpu for migrating timers
  984. * from an idle cpu. This is good for power-savings.
  985. *
  986. * We don't do similar optimization for completely idle system, as
  987. * selecting an idle cpu will add more delays to the timers than intended
  988. * (as that cpu's timer base may not be uptodate wrt jiffies etc).
  989. */
  990. int get_nohz_timer_target(void)
  991. {
  992. int cpu = smp_processor_id();
  993. int i;
  994. struct sched_domain *sd;
  995. for_each_domain(cpu, sd) {
  996. for_each_cpu(i, sched_domain_span(sd))
  997. if (!idle_cpu(i))
  998. return i;
  999. }
  1000. return cpu;
  1001. }
  1002. /*
  1003. * When add_timer_on() enqueues a timer into the timer wheel of an
  1004. * idle CPU then this timer might expire before the next timer event
  1005. * which is scheduled to wake up that CPU. In case of a completely
  1006. * idle system the next event might even be infinite time into the
  1007. * future. wake_up_idle_cpu() ensures that the CPU is woken up and
  1008. * leaves the inner idle loop so the newly added timer is taken into
  1009. * account when the CPU goes back to idle and evaluates the timer
  1010. * wheel for the next timer event.
  1011. */
  1012. void wake_up_idle_cpu(int cpu)
  1013. {
  1014. struct rq *rq = cpu_rq(cpu);
  1015. if (cpu == smp_processor_id())
  1016. return;
  1017. /*
  1018. * This is safe, as this function is called with the timer
  1019. * wheel base lock of (cpu) held. When the CPU is on the way
  1020. * to idle and has not yet set rq->curr to idle then it will
  1021. * be serialized on the timer wheel base lock and take the new
  1022. * timer into account automatically.
  1023. */
  1024. if (rq->curr != rq->idle)
  1025. return;
  1026. /*
  1027. * We can set TIF_RESCHED on the idle task of the other CPU
  1028. * lockless. The worst case is that the other CPU runs the
  1029. * idle task through an additional NOOP schedule()
  1030. */
  1031. set_tsk_need_resched(rq->idle);
  1032. /* NEED_RESCHED must be visible before we test polling */
  1033. smp_mb();
  1034. if (!tsk_is_polling(rq->idle))
  1035. smp_send_reschedule(cpu);
  1036. }
  1037. #endif /* CONFIG_NO_HZ */
  1038. static u64 sched_avg_period(void)
  1039. {
  1040. return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2;
  1041. }
  1042. static void sched_avg_update(struct rq *rq)
  1043. {
  1044. s64 period = sched_avg_period();
  1045. while ((s64)(rq->clock - rq->age_stamp) > period) {
  1046. /*
  1047. * Inline assembly required to prevent the compiler
  1048. * optimising this loop into a divmod call.
  1049. * See __iter_div_u64_rem() for another example of this.
  1050. */
  1051. asm("" : "+rm" (rq->age_stamp));
  1052. rq->age_stamp += period;
  1053. rq->rt_avg /= 2;
  1054. }
  1055. }
  1056. static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
  1057. {
  1058. rq->rt_avg += rt_delta;
  1059. sched_avg_update(rq);
  1060. }
  1061. #else /* !CONFIG_SMP */
  1062. static void resched_task(struct task_struct *p)
  1063. {
  1064. assert_raw_spin_locked(&task_rq(p)->lock);
  1065. set_tsk_need_resched(p);
  1066. }
  1067. static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
  1068. {
  1069. }
  1070. static void sched_avg_update(struct rq *rq)
  1071. {
  1072. }
  1073. #endif /* CONFIG_SMP */
  1074. #if BITS_PER_LONG == 32
  1075. # define WMULT_CONST (~0UL)
  1076. #else
  1077. # define WMULT_CONST (1UL << 32)
  1078. #endif
  1079. #define WMULT_SHIFT 32
  1080. /*
  1081. * Shift right and round:
  1082. */
  1083. #define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
  1084. /*
  1085. * delta *= weight / lw
  1086. */
  1087. static unsigned long
  1088. calc_delta_mine(unsigned long delta_exec, unsigned long weight,
  1089. struct load_weight *lw)
  1090. {
  1091. u64 tmp;
  1092. if (!lw->inv_weight) {
  1093. if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
  1094. lw->inv_weight = 1;
  1095. else
  1096. lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
  1097. / (lw->weight+1);
  1098. }
  1099. tmp = (u64)delta_exec * weight;
  1100. /*
  1101. * Check whether we'd overflow the 64-bit multiplication:
  1102. */
  1103. if (unlikely(tmp > WMULT_CONST))
  1104. tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
  1105. WMULT_SHIFT/2);
  1106. else
  1107. tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
  1108. return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
  1109. }
  1110. static inline void update_load_add(struct load_weight *lw, unsigned long inc)
  1111. {
  1112. lw->weight += inc;
  1113. lw->inv_weight = 0;
  1114. }
  1115. static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
  1116. {
  1117. lw->weight -= dec;
  1118. lw->inv_weight = 0;
  1119. }
  1120. static inline void update_load_set(struct load_weight *lw, unsigned long w)
  1121. {
  1122. lw->weight = w;
  1123. lw->inv_weight = 0;
  1124. }
  1125. /*
  1126. * To aid in avoiding the subversion of "niceness" due to uneven distribution
  1127. * of tasks with abnormal "nice" values across CPUs the contribution that
  1128. * each task makes to its run queue's load is weighted according to its
  1129. * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
  1130. * scaled version of the new time slice allocation that they receive on time
  1131. * slice expiry etc.
  1132. */
  1133. #define WEIGHT_IDLEPRIO 3
  1134. #define WMULT_IDLEPRIO 1431655765
  1135. /*
  1136. * Nice levels are multiplicative, with a gentle 10% change for every
  1137. * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
  1138. * nice 1, it will get ~10% less CPU time than another CPU-bound task
  1139. * that remained on nice 0.
  1140. *
  1141. * The "10% effect" is relative and cumulative: from _any_ nice level,
  1142. * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
  1143. * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
  1144. * If a task goes up by ~10% and another task goes down by ~10% then
  1145. * the relative distance between them is ~25%.)
  1146. */
  1147. static const int prio_to_weight[40] = {
  1148. /* -20 */ 88761, 71755, 56483, 46273, 36291,
  1149. /* -15 */ 29154, 23254, 18705, 14949, 11916,
  1150. /* -10 */ 9548, 7620, 6100, 4904, 3906,
  1151. /* -5 */ 3121, 2501, 1991, 1586, 1277,
  1152. /* 0 */ 1024, 820, 655, 526, 423,
  1153. /* 5 */ 335, 272, 215, 172, 137,
  1154. /* 10 */ 110, 87, 70, 56, 45,
  1155. /* 15 */ 36, 29, 23, 18, 15,
  1156. };
  1157. /*
  1158. * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
  1159. *
  1160. * In cases where the weight does not change often, we can use the
  1161. * precalculated inverse to speed up arithmetics by turning divisions
  1162. * into multiplications:
  1163. */
  1164. static const u32 prio_to_wmult[40] = {
  1165. /* -20 */ 48388, 59856, 76040, 92818, 118348,
  1166. /* -15 */ 147320, 184698, 229616, 287308, 360437,
  1167. /* -10 */ 449829, 563644, 704093, 875809, 1099582,
  1168. /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
  1169. /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
  1170. /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
  1171. /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
  1172. /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
  1173. };
  1174. /* Time spent by the tasks of the cpu accounting group executing in ... */
  1175. enum cpuacct_stat_index {
  1176. CPUACCT_STAT_USER, /* ... user mode */
  1177. CPUACCT_STAT_SYSTEM, /* ... kernel mode */
  1178. CPUACCT_STAT_NSTATS,
  1179. };
  1180. #ifdef CONFIG_CGROUP_CPUACCT
  1181. static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
  1182. static void cpuacct_update_stats(struct task_struct *tsk,
  1183. enum cpuacct_stat_index idx, cputime_t val);
  1184. #else
  1185. static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
  1186. static inline void cpuacct_update_stats(struct task_struct *tsk,
  1187. enum cpuacct_stat_index idx, cputime_t val) {}
  1188. #endif
  1189. static inline void inc_cpu_load(struct rq *rq, unsigned long load)
  1190. {
  1191. update_load_add(&rq->load, load);
  1192. }
  1193. static inline void dec_cpu_load(struct rq *rq, unsigned long load)
  1194. {
  1195. update_load_sub(&rq->load, load);
  1196. }
  1197. #if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) || defined(CONFIG_RT_GROUP_SCHED)
  1198. typedef int (*tg_visitor)(struct task_group *, void *);
  1199. /*
  1200. * Iterate the full tree, calling @down when first entering a node and @up when
  1201. * leaving it for the final time.
  1202. */
  1203. static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
  1204. {
  1205. struct task_group *parent, *child;
  1206. int ret;
  1207. rcu_read_lock();
  1208. parent = &root_task_group;
  1209. down:
  1210. ret = (*down)(parent, data);
  1211. if (ret)
  1212. goto out_unlock;
  1213. list_for_each_entry_rcu(child, &parent->children, siblings) {
  1214. parent = child;
  1215. goto down;
  1216. up:
  1217. continue;
  1218. }
  1219. ret = (*up)(parent, data);
  1220. if (ret)
  1221. goto out_unlock;
  1222. child = parent;
  1223. parent = parent->parent;
  1224. if (parent)
  1225. goto up;
  1226. out_unlock:
  1227. rcu_read_unlock();
  1228. return ret;
  1229. }
  1230. static int tg_nop(struct task_group *tg, void *data)
  1231. {
  1232. return 0;
  1233. }
  1234. #endif
  1235. #ifdef CONFIG_SMP
  1236. /* Used instead of source_load when we know the type == 0 */
  1237. static unsigned long weighted_cpuload(const int cpu)
  1238. {
  1239. return cpu_rq(cpu)->load.weight;
  1240. }
  1241. /*
  1242. * Return a low guess at the load of a migration-source cpu weighted
  1243. * according to the scheduling class and "nice" value.
  1244. *
  1245. * We want to under-estimate the load of migration sources, to
  1246. * balance conservatively.
  1247. */
  1248. static unsigned long source_load(int cpu, int type)
  1249. {
  1250. struct rq *rq = cpu_rq(cpu);
  1251. unsigned long total = weighted_cpuload(cpu);
  1252. if (type == 0 || !sched_feat(LB_BIAS))
  1253. return total;
  1254. return min(rq->cpu_load[type-1], total);
  1255. }
  1256. /*
  1257. * Return a high guess at the load of a migration-target cpu weighted
  1258. * according to the scheduling class and "nice" value.
  1259. */
  1260. static unsigned long target_load(int cpu, int type)
  1261. {
  1262. struct rq *rq = cpu_rq(cpu);
  1263. unsigned long total = weighted_cpuload(cpu);
  1264. if (type == 0 || !sched_feat(LB_BIAS))
  1265. return total;
  1266. return max(rq->cpu_load[type-1], total);
  1267. }
  1268. static unsigned long power_of(int cpu)
  1269. {
  1270. return cpu_rq(cpu)->cpu_power;
  1271. }
  1272. static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
  1273. static unsigned long cpu_avg_load_per_task(int cpu)
  1274. {
  1275. struct rq *rq = cpu_rq(cpu);
  1276. unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
  1277. if (nr_running)
  1278. rq->avg_load_per_task = rq->load.weight / nr_running;
  1279. else
  1280. rq->avg_load_per_task = 0;
  1281. return rq->avg_load_per_task;
  1282. }
  1283. #ifdef CONFIG_FAIR_GROUP_SCHED
  1284. /*
  1285. * Compute the cpu's hierarchical load factor for each task group.
  1286. * This needs to be done in a top-down fashion because the load of a child
  1287. * group is a fraction of its parents load.
  1288. */
  1289. static int tg_load_down(struct task_group *tg, void *data)
  1290. {
  1291. unsigned long load;
  1292. long cpu = (long)data;
  1293. if (!tg->parent) {
  1294. load = cpu_rq(cpu)->load.weight;
  1295. } else {
  1296. load = tg->parent->cfs_rq[cpu]->h_load;
  1297. load *= tg->se[cpu]->load.weight;
  1298. load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
  1299. }
  1300. tg->cfs_rq[cpu]->h_load = load;
  1301. return 0;
  1302. }
  1303. static void update_h_load(long cpu)
  1304. {
  1305. walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
  1306. }
  1307. #endif
  1308. #ifdef CONFIG_PREEMPT
  1309. static void double_rq_lock(struct rq *rq1, struct rq *rq2);
  1310. /*
  1311. * fair double_lock_balance: Safely acquires both rq->locks in a fair
  1312. * way at the expense of forcing extra atomic operations in all
  1313. * invocations. This assures that the double_lock is acquired using the
  1314. * same underlying policy as the spinlock_t on this architecture, which
  1315. * reduces latency compared to the unfair variant below. However, it
  1316. * also adds more overhead and therefore may reduce throughput.
  1317. */
  1318. static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1319. __releases(this_rq->lock)
  1320. __acquires(busiest->lock)
  1321. __acquires(this_rq->lock)
  1322. {
  1323. raw_spin_unlock(&this_rq->lock);
  1324. double_rq_lock(this_rq, busiest);
  1325. return 1;
  1326. }
  1327. #else
  1328. /*
  1329. * Unfair double_lock_balance: Optimizes throughput at the expense of
  1330. * latency by eliminating extra atomic operations when the locks are
  1331. * already in proper order on entry. This favors lower cpu-ids and will
  1332. * grant the double lock to lower cpus over higher ids under contention,
  1333. * regardless of entry order into the function.
  1334. */
  1335. static int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1336. __releases(this_rq->lock)
  1337. __acquires(busiest->lock)
  1338. __acquires(this_rq->lock)
  1339. {
  1340. int ret = 0;
  1341. if (unlikely(!raw_spin_trylock(&busiest->lock))) {
  1342. if (busiest < this_rq) {
  1343. raw_spin_unlock(&this_rq->lock);
  1344. raw_spin_lock(&busiest->lock);
  1345. raw_spin_lock_nested(&this_rq->lock,
  1346. SINGLE_DEPTH_NESTING);
  1347. ret = 1;
  1348. } else
  1349. raw_spin_lock_nested(&busiest->lock,
  1350. SINGLE_DEPTH_NESTING);
  1351. }
  1352. return ret;
  1353. }
  1354. #endif /* CONFIG_PREEMPT */
  1355. /*
  1356. * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
  1357. */
  1358. static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1359. {
  1360. if (unlikely(!irqs_disabled())) {
  1361. /* printk() doesn't work good under rq->lock */
  1362. raw_spin_unlock(&this_rq->lock);
  1363. BUG_ON(1);
  1364. }
  1365. return _double_lock_balance(this_rq, busiest);
  1366. }
  1367. static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
  1368. __releases(busiest->lock)
  1369. {
  1370. raw_spin_unlock(&busiest->lock);
  1371. lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
  1372. }
  1373. /*
  1374. * double_rq_lock - safely lock two runqueues
  1375. *
  1376. * Note this does not disable interrupts like task_rq_lock,
  1377. * you need to do so manually before calling.
  1378. */
  1379. static void double_rq_lock(struct rq *rq1, struct rq *rq2)
  1380. __acquires(rq1->lock)
  1381. __acquires(rq2->lock)
  1382. {
  1383. BUG_ON(!irqs_disabled());
  1384. if (rq1 == rq2) {
  1385. raw_spin_lock(&rq1->lock);
  1386. __acquire(rq2->lock); /* Fake it out ;) */
  1387. } else {
  1388. if (rq1 < rq2) {
  1389. raw_spin_lock(&rq1->lock);
  1390. raw_spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
  1391. } else {
  1392. raw_spin_lock(&rq2->lock);
  1393. raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
  1394. }
  1395. }
  1396. }
  1397. /*
  1398. * double_rq_unlock - safely unlock two runqueues
  1399. *
  1400. * Note this does not restore interrupts like task_rq_unlock,
  1401. * you need to do so manually after calling.
  1402. */
  1403. static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
  1404. __releases(rq1->lock)
  1405. __releases(rq2->lock)
  1406. {
  1407. raw_spin_unlock(&rq1->lock);
  1408. if (rq1 != rq2)
  1409. raw_spin_unlock(&rq2->lock);
  1410. else
  1411. __release(rq2->lock);
  1412. }
  1413. #endif
  1414. static void calc_load_account_idle(struct rq *this_rq);
  1415. static void update_sysctl(void);
  1416. static int get_update_sysctl_factor(void);
  1417. static void update_cpu_load(struct rq *this_rq);
  1418. static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
  1419. {
  1420. set_task_rq(p, cpu);
  1421. #ifdef CONFIG_SMP
  1422. /*
  1423. * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
  1424. * successfuly executed on another CPU. We must ensure that updates of
  1425. * per-task data have been completed by this moment.
  1426. */
  1427. smp_wmb();
  1428. task_thread_info(p)->cpu = cpu;
  1429. #endif
  1430. }
  1431. static const struct sched_class rt_sched_class;
  1432. #define sched_class_highest (&stop_sched_class)
  1433. #define for_each_class(class) \
  1434. for (class = sched_class_highest; class; class = class->next)
  1435. #include "sched_stats.h"
  1436. static void inc_nr_running(struct rq *rq)
  1437. {
  1438. rq->nr_running++;
  1439. }
  1440. static void dec_nr_running(struct rq *rq)
  1441. {
  1442. rq->nr_running--;
  1443. }
  1444. static void set_load_weight(struct task_struct *p)
  1445. {
  1446. /*
  1447. * SCHED_IDLE tasks get minimal weight:
  1448. */
  1449. if (p->policy == SCHED_IDLE) {
  1450. p->se.load.weight = WEIGHT_IDLEPRIO;
  1451. p->se.load.inv_weight = WMULT_IDLEPRIO;
  1452. return;
  1453. }
  1454. p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
  1455. p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
  1456. }
  1457. static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
  1458. {
  1459. update_rq_clock(rq);
  1460. sched_info_queued(p);
  1461. p->sched_class->enqueue_task(rq, p, flags);
  1462. p->se.on_rq = 1;
  1463. }
  1464. static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
  1465. {
  1466. update_rq_clock(rq);
  1467. sched_info_dequeued(p);
  1468. p->sched_class->dequeue_task(rq, p, flags);
  1469. p->se.on_rq = 0;
  1470. }
  1471. /*
  1472. * activate_task - move a task to the runqueue.
  1473. */
  1474. static void activate_task(struct rq *rq, struct task_struct *p, int flags)
  1475. {
  1476. if (task_contributes_to_load(p))
  1477. rq->nr_uninterruptible--;
  1478. enqueue_task(rq, p, flags);
  1479. inc_nr_running(rq);
  1480. }
  1481. /*
  1482. * deactivate_task - remove a task from the runqueue.
  1483. */
  1484. static void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
  1485. {
  1486. if (task_contributes_to_load(p))
  1487. rq->nr_uninterruptible++;
  1488. dequeue_task(rq, p, flags);
  1489. dec_nr_running(rq);
  1490. }
  1491. #ifdef CONFIG_IRQ_TIME_ACCOUNTING
  1492. /*
  1493. * There are no locks covering percpu hardirq/softirq time.
  1494. * They are only modified in account_system_vtime, on corresponding CPU
  1495. * with interrupts disabled. So, writes are safe.
  1496. * They are read and saved off onto struct rq in update_rq_clock().
  1497. * This may result in other CPU reading this CPU's irq time and can
  1498. * race with irq/account_system_vtime on this CPU. We would either get old
  1499. * or new value with a side effect of accounting a slice of irq time to wrong
  1500. * task when irq is in progress while we read rq->clock. That is a worthy
  1501. * compromise in place of having locks on each irq in account_system_time.
  1502. */
  1503. static DEFINE_PER_CPU(u64, cpu_hardirq_time);
  1504. static DEFINE_PER_CPU(u64, cpu_softirq_time);
  1505. static DEFINE_PER_CPU(u64, irq_start_time);
  1506. static int sched_clock_irqtime;
  1507. void enable_sched_clock_irqtime(void)
  1508. {
  1509. sched_clock_irqtime = 1;
  1510. }
  1511. void disable_sched_clock_irqtime(void)
  1512. {
  1513. sched_clock_irqtime = 0;
  1514. }
  1515. #ifndef CONFIG_64BIT
  1516. static DEFINE_PER_CPU(seqcount_t, irq_time_seq);
  1517. static inline void irq_time_write_begin(void)
  1518. {
  1519. __this_cpu_inc(irq_time_seq.sequence);
  1520. smp_wmb();
  1521. }
  1522. static inline void irq_time_write_end(void)
  1523. {
  1524. smp_wmb();
  1525. __this_cpu_inc(irq_time_seq.sequence);
  1526. }
  1527. static inline u64 irq_time_read(int cpu)
  1528. {
  1529. u64 irq_time;
  1530. unsigned seq;
  1531. do {
  1532. seq = read_seqcount_begin(&per_cpu(irq_time_seq, cpu));
  1533. irq_time = per_cpu(cpu_softirq_time, cpu) +
  1534. per_cpu(cpu_hardirq_time, cpu);
  1535. } while (read_seqcount_retry(&per_cpu(irq_time_seq, cpu), seq));
  1536. return irq_time;
  1537. }
  1538. #else /* CONFIG_64BIT */
  1539. static inline void irq_time_write_begin(void)
  1540. {
  1541. }
  1542. static inline void irq_time_write_end(void)
  1543. {
  1544. }
  1545. static inline u64 irq_time_read(int cpu)
  1546. {
  1547. return per_cpu(cpu_softirq_time, cpu) + per_cpu(cpu_hardirq_time, cpu);
  1548. }
  1549. #endif /* CONFIG_64BIT */
  1550. /*
  1551. * Called before incrementing preempt_count on {soft,}irq_enter
  1552. * and before decrementing preempt_count on {soft,}irq_exit.
  1553. */
  1554. void account_system_vtime(struct task_struct *curr)
  1555. {
  1556. unsigned long flags;
  1557. s64 delta;
  1558. int cpu;
  1559. if (!sched_clock_irqtime)
  1560. return;
  1561. local_irq_save(flags);
  1562. cpu = smp_processor_id();
  1563. delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time);
  1564. __this_cpu_add(irq_start_time, delta);
  1565. irq_time_write_begin();
  1566. /*
  1567. * We do not account for softirq time from ksoftirqd here.
  1568. * We want to continue accounting softirq time to ksoftirqd thread
  1569. * in that case, so as not to confuse scheduler with a special task
  1570. * that do not consume any time, but still wants to run.
  1571. */
  1572. if (hardirq_count())
  1573. __this_cpu_add(cpu_hardirq_time, delta);
  1574. else if (in_serving_softirq() && !(curr->flags & PF_KSOFTIRQD))
  1575. __this_cpu_add(cpu_softirq_time, delta);
  1576. irq_time_write_end();
  1577. local_irq_restore(flags);
  1578. }
  1579. EXPORT_SYMBOL_GPL(account_system_vtime);
  1580. static void update_rq_clock_task(struct rq *rq, s64 delta)
  1581. {
  1582. s64 irq_delta;
  1583. irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
  1584. /*
  1585. * Since irq_time is only updated on {soft,}irq_exit, we might run into
  1586. * this case when a previous update_rq_clock() happened inside a
  1587. * {soft,}irq region.
  1588. *
  1589. * When this happens, we stop ->clock_task and only update the
  1590. * prev_irq_time stamp to account for the part that fit, so that a next
  1591. * update will consume the rest. This ensures ->clock_task is
  1592. * monotonic.
  1593. *
  1594. * It does however cause some slight miss-attribution of {soft,}irq
  1595. * time, a more accurate solution would be to update the irq_time using
  1596. * the current rq->clock timestamp, except that would require using
  1597. * atomic ops.
  1598. */
  1599. if (irq_delta > delta)
  1600. irq_delta = delta;
  1601. rq->prev_irq_time += irq_delta;
  1602. delta -= irq_delta;
  1603. rq->clock_task += delta;
  1604. if (irq_delta && sched_feat(NONIRQ_POWER))
  1605. sched_rt_avg_update(rq, irq_delta);
  1606. }
  1607. #else /* CONFIG_IRQ_TIME_ACCOUNTING */
  1608. static void update_rq_clock_task(struct rq *rq, s64 delta)
  1609. {
  1610. rq->clock_task += delta;
  1611. }
  1612. #endif /* CONFIG_IRQ_TIME_ACCOUNTING */
  1613. #include "sched_idletask.c"
  1614. #include "sched_fair.c"
  1615. #include "sched_rt.c"
  1616. #include "sched_autogroup.c"
  1617. #include "sched_stoptask.c"
  1618. #ifdef CONFIG_SCHED_DEBUG
  1619. # include "sched_debug.c"
  1620. #endif
  1621. void sched_set_stop_task(int cpu, struct task_struct *stop)
  1622. {
  1623. struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
  1624. struct task_struct *old_stop = cpu_rq(cpu)->stop;
  1625. if (stop) {
  1626. /*
  1627. * Make it appear like a SCHED_FIFO task, its something
  1628. * userspace knows about and won't get confused about.
  1629. *
  1630. * Also, it will make PI more or less work without too
  1631. * much confusion -- but then, stop work should not
  1632. * rely on PI working anyway.
  1633. */
  1634. sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
  1635. stop->sched_class = &stop_sched_class;
  1636. }
  1637. cpu_rq(cpu)->stop = stop;
  1638. if (old_stop) {
  1639. /*
  1640. * Reset it back to a normal scheduling class so that
  1641. * it can die in pieces.
  1642. */
  1643. old_stop->sched_class = &rt_sched_class;
  1644. }
  1645. }
  1646. /*
  1647. * __normal_prio - return the priority that is based on the static prio
  1648. */
  1649. static inline int __normal_prio(struct task_struct *p)
  1650. {
  1651. return p->static_prio;
  1652. }
  1653. /*
  1654. * Calculate the expected normal priority: i.e. priority
  1655. * without taking RT-inheritance into account. Might be
  1656. * boosted by interactivity modifiers. Changes upon fork,
  1657. * setprio syscalls, and whenever the interactivity
  1658. * estimator recalculates.
  1659. */
  1660. static inline int normal_prio(struct task_struct *p)
  1661. {
  1662. int prio;
  1663. if (task_has_rt_policy(p))
  1664. prio = MAX_RT_PRIO-1 - p->rt_priority;
  1665. else
  1666. prio = __normal_prio(p);
  1667. return prio;
  1668. }
  1669. /*
  1670. * Calculate the current priority, i.e. the priority
  1671. * taken into account by the scheduler. This value might
  1672. * be boosted by RT tasks, or might be boosted by
  1673. * interactivity modifiers. Will be RT if the task got
  1674. * RT-boosted. If not then it returns p->normal_prio.
  1675. */
  1676. static int effective_prio(struct task_struct *p)
  1677. {
  1678. p->normal_prio = normal_prio(p);
  1679. /*
  1680. * If we are RT tasks or we were boosted to RT priority,
  1681. * keep the priority unchanged. Otherwise, update priority
  1682. * to the normal priority:
  1683. */
  1684. if (!rt_prio(p->prio))
  1685. return p->normal_prio;
  1686. return p->prio;
  1687. }
  1688. /**
  1689. * task_curr - is this task currently executing on a CPU?
  1690. * @p: the task in question.
  1691. */
  1692. inline int task_curr(const struct task_struct *p)
  1693. {
  1694. return cpu_curr(task_cpu(p)) == p;
  1695. }
  1696. static inline void check_class_changed(struct rq *rq, struct task_struct *p,
  1697. const struct sched_class *prev_class,
  1698. int oldprio, int running)
  1699. {
  1700. if (prev_class != p->sched_class) {
  1701. if (prev_class->switched_from)
  1702. prev_class->switched_from(rq, p, running);
  1703. p->sched_class->switched_to(rq, p, running);
  1704. } else
  1705. p->sched_class->prio_changed(rq, p, oldprio, running);
  1706. }
  1707. static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
  1708. {
  1709. const struct sched_class *class;
  1710. if (p->sched_class == rq->curr->sched_class) {
  1711. rq->curr->sched_class->check_preempt_curr(rq, p, flags);
  1712. } else {
  1713. for_each_class(class) {
  1714. if (class == rq->curr->sched_class)
  1715. break;
  1716. if (class == p->sched_class) {
  1717. resched_task(rq->curr);
  1718. break;
  1719. }
  1720. }
  1721. }
  1722. /*
  1723. * A queue event has occurred, and we're going to schedule. In
  1724. * this case, we can save a useless back to back clock update.
  1725. */
  1726. if (rq->curr->se.on_rq && test_tsk_need_resched(rq->curr))
  1727. rq->skip_clock_update = 1;
  1728. }
  1729. #ifdef CONFIG_SMP
  1730. /*
  1731. * Is this task likely cache-hot:
  1732. */
  1733. static int
  1734. task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
  1735. {
  1736. s64 delta;
  1737. if (p->sched_class != &fair_sched_class)
  1738. return 0;
  1739. if (unlikely(p->policy == SCHED_IDLE))
  1740. return 0;
  1741. /*
  1742. * Buddy candidates are cache hot:
  1743. */
  1744. if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running &&
  1745. (&p->se == cfs_rq_of(&p->se)->next ||
  1746. &p->se == cfs_rq_of(&p->se)->last))
  1747. return 1;
  1748. if (sysctl_sched_migration_cost == -1)
  1749. return 1;
  1750. if (sysctl_sched_migration_cost == 0)
  1751. return 0;
  1752. delta = now - p->se.exec_start;
  1753. return delta < (s64)sysctl_sched_migration_cost;
  1754. }
  1755. void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
  1756. {
  1757. #ifdef CONFIG_SCHED_DEBUG
  1758. /*
  1759. * We should never call set_task_cpu() on a blocked task,
  1760. * ttwu() will sort out the placement.
  1761. */
  1762. WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
  1763. !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE));
  1764. #endif
  1765. trace_sched_migrate_task(p, new_cpu);
  1766. if (task_cpu(p) != new_cpu) {
  1767. p->se.nr_migrations++;
  1768. perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 1, NULL, 0);
  1769. }
  1770. __set_task_cpu(p, new_cpu);
  1771. }
  1772. struct migration_arg {
  1773. struct task_struct *task;
  1774. int dest_cpu;
  1775. };
  1776. static int migration_cpu_stop(void *data);
  1777. /*
  1778. * The task's runqueue lock must be held.
  1779. * Returns true if you have to wait for migration thread.
  1780. */
  1781. static bool migrate_task(struct task_struct *p, struct rq *rq)
  1782. {
  1783. /*
  1784. * If the task is not on a runqueue (and not running), then
  1785. * the next wake-up will properly place the task.
  1786. */
  1787. return p->se.on_rq || task_running(rq, p);
  1788. }
  1789. /*
  1790. * wait_task_inactive - wait for a thread to unschedule.
  1791. *
  1792. * If @match_state is nonzero, it's the @p->state value just checked and
  1793. * not expected to change. If it changes, i.e. @p might have woken up,
  1794. * then return zero. When we succeed in waiting for @p to be off its CPU,
  1795. * we return a positive number (its total switch count). If a second call
  1796. * a short while later returns the same number, the caller can be sure that
  1797. * @p has remained unscheduled the whole time.
  1798. *
  1799. * The caller must ensure that the task *will* unschedule sometime soon,
  1800. * else this function might spin for a *long* time. This function can't
  1801. * be called with interrupts off, or it may introduce deadlock with
  1802. * smp_call_function() if an IPI is sent by the same process we are
  1803. * waiting to become inactive.
  1804. */
  1805. unsigned long wait_task_inactive(struct task_struct *p, long match_state)
  1806. {
  1807. unsigned long flags;
  1808. int running, on_rq;
  1809. unsigned long ncsw;
  1810. struct rq *rq;
  1811. for (;;) {
  1812. /*
  1813. * We do the initial early heuristics without holding
  1814. * any task-queue locks at all. We'll only try to get
  1815. * the runqueue lock when things look like they will
  1816. * work out!
  1817. */
  1818. rq = task_rq(p);
  1819. /*
  1820. * If the task is actively running on another CPU
  1821. * still, just relax and busy-wait without holding
  1822. * any locks.
  1823. *
  1824. * NOTE! Since we don't hold any locks, it's not
  1825. * even sure that "rq" stays as the right runqueue!
  1826. * But we don't care, since "task_running()" will
  1827. * return false if the runqueue has changed and p
  1828. * is actually now running somewhere else!
  1829. */
  1830. while (task_running(rq, p)) {
  1831. if (match_state && unlikely(p->state != match_state))
  1832. return 0;
  1833. cpu_relax();
  1834. }
  1835. /*
  1836. * Ok, time to look more closely! We need the rq
  1837. * lock now, to be *sure*. If we're wrong, we'll
  1838. * just go back and repeat.
  1839. */
  1840. rq = task_rq_lock(p, &flags);
  1841. trace_sched_wait_task(p);
  1842. running = task_running(rq, p);
  1843. on_rq = p->se.on_rq;
  1844. ncsw = 0;
  1845. if (!match_state || p->state == match_state)
  1846. ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
  1847. task_rq_unlock(rq, &flags);
  1848. /*
  1849. * If it changed from the expected state, bail out now.
  1850. */
  1851. if (unlikely(!ncsw))
  1852. break;
  1853. /*
  1854. * Was it really running after all now that we
  1855. * checked with the proper locks actually held?
  1856. *
  1857. * Oops. Go back and try again..
  1858. */
  1859. if (unlikely(running)) {
  1860. cpu_relax();
  1861. continue;
  1862. }
  1863. /*
  1864. * It's not enough that it's not actively running,
  1865. * it must be off the runqueue _entirely_, and not
  1866. * preempted!
  1867. *
  1868. * So if it was still runnable (but just not actively
  1869. * running right now), it's preempted, and we should
  1870. * yield - it could be a while.
  1871. */
  1872. if (unlikely(on_rq)) {
  1873. schedule_timeout_uninterruptible(1);
  1874. continue;
  1875. }
  1876. /*
  1877. * Ahh, all good. It wasn't running, and it wasn't
  1878. * runnable, which means that it will never become
  1879. * running in the future either. We're all done!
  1880. */
  1881. break;
  1882. }
  1883. return ncsw;
  1884. }
  1885. /***
  1886. * kick_process - kick a running thread to enter/exit the kernel
  1887. * @p: the to-be-kicked thread
  1888. *
  1889. * Cause a process which is running on another CPU to enter
  1890. * kernel-mode, without any delay. (to get signals handled.)
  1891. *
  1892. * NOTE: this function doesnt have to take the runqueue lock,
  1893. * because all it wants to ensure is that the remote task enters
  1894. * the kernel. If the IPI races and the task has been migrated
  1895. * to another CPU then no harm is done and the purpose has been
  1896. * achieved as well.
  1897. */
  1898. void kick_process(struct task_struct *p)
  1899. {
  1900. int cpu;
  1901. preempt_disable();
  1902. cpu = task_cpu(p);
  1903. if ((cpu != smp_processor_id()) && task_curr(p))
  1904. smp_send_reschedule(cpu);
  1905. preempt_enable();
  1906. }
  1907. EXPORT_SYMBOL_GPL(kick_process);
  1908. #endif /* CONFIG_SMP */
  1909. /**
  1910. * task_oncpu_function_call - call a function on the cpu on which a task runs
  1911. * @p: the task to evaluate
  1912. * @func: the function to be called
  1913. * @info: the function call argument
  1914. *
  1915. * Calls the function @func when the task is currently running. This might
  1916. * be on the current CPU, which just calls the function directly
  1917. */
  1918. void task_oncpu_function_call(struct task_struct *p,
  1919. void (*func) (void *info), void *info)
  1920. {
  1921. int cpu;
  1922. preempt_disable();
  1923. cpu = task_cpu(p);
  1924. if (task_curr(p))
  1925. smp_call_function_single(cpu, func, info, 1);
  1926. preempt_enable();
  1927. }
  1928. #ifdef CONFIG_SMP
  1929. /*
  1930. * ->cpus_allowed is protected by either TASK_WAKING or rq->lock held.
  1931. */
  1932. static int select_fallback_rq(int cpu, struct task_struct *p)
  1933. {
  1934. int dest_cpu;
  1935. const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(cpu));
  1936. /* Look for allowed, online CPU in same node. */
  1937. for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
  1938. if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
  1939. return dest_cpu;
  1940. /* Any allowed, online CPU? */
  1941. dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
  1942. if (dest_cpu < nr_cpu_ids)
  1943. return dest_cpu;
  1944. /* No more Mr. Nice Guy. */
  1945. dest_cpu = cpuset_cpus_allowed_fallback(p);
  1946. /*
  1947. * Don't tell them about moving exiting tasks or
  1948. * kernel threads (both mm NULL), since they never
  1949. * leave kernel.
  1950. */
  1951. if (p->mm && printk_ratelimit()) {
  1952. printk(KERN_INFO "process %d (%s) no longer affine to cpu%d\n",
  1953. task_pid_nr(p), p->comm, cpu);
  1954. }
  1955. return dest_cpu;
  1956. }
  1957. /*
  1958. * The caller (fork, wakeup) owns TASK_WAKING, ->cpus_allowed is stable.
  1959. */
  1960. static inline
  1961. int select_task_rq(struct rq *rq, struct task_struct *p, int sd_flags, int wake_flags)
  1962. {
  1963. int cpu = p->sched_class->select_task_rq(rq, p, sd_flags, wake_flags);
  1964. /*
  1965. * In order not to call set_task_cpu() on a blocking task we need
  1966. * to rely on ttwu() to place the task on a valid ->cpus_allowed
  1967. * cpu.
  1968. *
  1969. * Since this is common to all placement strategies, this lives here.
  1970. *
  1971. * [ this allows ->select_task() to simply return task_cpu(p) and
  1972. * not worry about this generic constraint ]
  1973. */
  1974. if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed) ||
  1975. !cpu_online(cpu)))
  1976. cpu = select_fallback_rq(task_cpu(p), p);
  1977. return cpu;
  1978. }
  1979. static void update_avg(u64 *avg, u64 sample)
  1980. {
  1981. s64 diff = sample - *avg;
  1982. *avg += diff >> 3;
  1983. }
  1984. #endif
  1985. static inline void ttwu_activate(struct task_struct *p, struct rq *rq,
  1986. bool is_sync, bool is_migrate, bool is_local,
  1987. unsigned long en_flags)
  1988. {
  1989. schedstat_inc(p, se.statistics.nr_wakeups);
  1990. if (is_sync)
  1991. schedstat_inc(p, se.statistics.nr_wakeups_sync);
  1992. if (is_migrate)
  1993. schedstat_inc(p, se.statistics.nr_wakeups_migrate);
  1994. if (is_local)
  1995. schedstat_inc(p, se.statistics.nr_wakeups_local);
  1996. else
  1997. schedstat_inc(p, se.statistics.nr_wakeups_remote);
  1998. activate_task(rq, p, en_flags);
  1999. }
  2000. static inline void ttwu_post_activation(struct task_struct *p, struct rq *rq,
  2001. int wake_flags, bool success)
  2002. {
  2003. trace_sched_wakeup(p, success);
  2004. check_preempt_curr(rq, p, wake_flags);
  2005. p->state = TASK_RUNNING;
  2006. #ifdef CONFIG_SMP
  2007. if (p->sched_class->task_woken)
  2008. p->sched_class->task_woken(rq, p);
  2009. if (unlikely(rq->idle_stamp)) {
  2010. u64 delta = rq->clock - rq->idle_stamp;
  2011. u64 max = 2*sysctl_sched_migration_cost;
  2012. if (delta > max)
  2013. rq->avg_idle = max;
  2014. else
  2015. update_avg(&rq->avg_idle, delta);
  2016. rq->idle_stamp = 0;
  2017. }
  2018. #endif
  2019. /* if a worker is waking up, notify workqueue */
  2020. if ((p->flags & PF_WQ_WORKER) && success)
  2021. wq_worker_waking_up(p, cpu_of(rq));
  2022. }
  2023. /**
  2024. * try_to_wake_up - wake up a thread
  2025. * @p: the thread to be awakened
  2026. * @state: the mask of task states that can be woken
  2027. * @wake_flags: wake modifier flags (WF_*)
  2028. *
  2029. * Put it on the run-queue if it's not already there. The "current"
  2030. * thread is always on the run-queue (except when the actual
  2031. * re-schedule is in progress), and as such you're allowed to do
  2032. * the simpler "current->state = TASK_RUNNING" to mark yourself
  2033. * runnable without the overhead of this.
  2034. *
  2035. * Returns %true if @p was woken up, %false if it was already running
  2036. * or @state didn't match @p's state.
  2037. */
  2038. static int try_to_wake_up(struct task_struct *p, unsigned int state,
  2039. int wake_flags)
  2040. {
  2041. int cpu, orig_cpu, this_cpu, success = 0;
  2042. unsigned long flags;
  2043. unsigned long en_flags = ENQUEUE_WAKEUP;
  2044. struct rq *rq;
  2045. this_cpu = get_cpu();
  2046. smp_wmb();
  2047. rq = task_rq_lock(p, &flags);
  2048. if (!(p->state & state))
  2049. goto out;
  2050. if (p->se.on_rq)
  2051. goto out_running;
  2052. cpu = task_cpu(p);
  2053. orig_cpu = cpu;
  2054. #ifdef CONFIG_SMP
  2055. if (unlikely(task_running(rq, p)))
  2056. goto out_activate;
  2057. /*
  2058. * In order to handle concurrent wakeups and release the rq->lock
  2059. * we put the task in TASK_WAKING state.
  2060. *
  2061. * First fix up the nr_uninterruptible count:
  2062. */
  2063. if (task_contributes_to_load(p)) {
  2064. if (likely(cpu_online(orig_cpu)))
  2065. rq->nr_uninterruptible--;
  2066. else
  2067. this_rq()->nr_uninterruptible--;
  2068. }
  2069. p->state = TASK_WAKING;
  2070. if (p->sched_class->task_waking) {
  2071. p->sched_class->task_waking(rq, p);
  2072. en_flags |= ENQUEUE_WAKING;
  2073. }
  2074. cpu = select_task_rq(rq, p, SD_BALANCE_WAKE, wake_flags);
  2075. if (cpu != orig_cpu)
  2076. set_task_cpu(p, cpu);
  2077. __task_rq_unlock(rq);
  2078. rq = cpu_rq(cpu);
  2079. raw_spin_lock(&rq->lock);
  2080. /*
  2081. * We migrated the task without holding either rq->lock, however
  2082. * since the task is not on the task list itself, nobody else
  2083. * will try and migrate the task, hence the rq should match the
  2084. * cpu we just moved it to.
  2085. */
  2086. WARN_ON(task_cpu(p) != cpu);
  2087. WARN_ON(p->state != TASK_WAKING);
  2088. #ifdef CONFIG_SCHEDSTATS
  2089. schedstat_inc(rq, ttwu_count);
  2090. if (cpu == this_cpu)
  2091. schedstat_inc(rq, ttwu_local);
  2092. else {
  2093. struct sched_domain *sd;
  2094. for_each_domain(this_cpu, sd) {
  2095. if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
  2096. schedstat_inc(sd, ttwu_wake_remote);
  2097. break;
  2098. }
  2099. }
  2100. }
  2101. #endif /* CONFIG_SCHEDSTATS */
  2102. out_activate:
  2103. #endif /* CONFIG_SMP */
  2104. ttwu_activate(p, rq, wake_flags & WF_SYNC, orig_cpu != cpu,
  2105. cpu == this_cpu, en_flags);
  2106. success = 1;
  2107. out_running:
  2108. ttwu_post_activation(p, rq, wake_flags, success);
  2109. out:
  2110. task_rq_unlock(rq, &flags);
  2111. put_cpu();
  2112. return success;
  2113. }
  2114. /**
  2115. * try_to_wake_up_local - try to wake up a local task with rq lock held
  2116. * @p: the thread to be awakened
  2117. *
  2118. * Put @p on the run-queue if it's not alredy there. The caller must
  2119. * ensure that this_rq() is locked, @p is bound to this_rq() and not
  2120. * the current task. this_rq() stays locked over invocation.
  2121. */
  2122. static void try_to_wake_up_local(struct task_struct *p)
  2123. {
  2124. struct rq *rq = task_rq(p);
  2125. bool success = false;
  2126. BUG_ON(rq != this_rq());
  2127. BUG_ON(p == current);
  2128. lockdep_assert_held(&rq->lock);
  2129. if (!(p->state & TASK_NORMAL))
  2130. return;
  2131. if (!p->se.on_rq) {
  2132. if (likely(!task_running(rq, p))) {
  2133. schedstat_inc(rq, ttwu_count);
  2134. schedstat_inc(rq, ttwu_local);
  2135. }
  2136. ttwu_activate(p, rq, false, false, true, ENQUEUE_WAKEUP);
  2137. success = true;
  2138. }
  2139. ttwu_post_activation(p, rq, 0, success);
  2140. }
  2141. /**
  2142. * wake_up_process - Wake up a specific process
  2143. * @p: The process to be woken up.
  2144. *
  2145. * Attempt to wake up the nominated process and move it to the set of runnable
  2146. * processes. Returns 1 if the process was woken up, 0 if it was already
  2147. * running.
  2148. *
  2149. * It may be assumed that this function implies a write memory barrier before
  2150. * changing the task state if and only if any tasks are woken up.
  2151. */
  2152. int wake_up_process(struct task_struct *p)
  2153. {
  2154. return try_to_wake_up(p, TASK_ALL, 0);
  2155. }
  2156. EXPORT_SYMBOL(wake_up_process);
  2157. int wake_up_state(struct task_struct *p, unsigned int state)
  2158. {
  2159. return try_to_wake_up(p, state, 0);
  2160. }
  2161. /*
  2162. * Perform scheduler related setup for a newly forked process p.
  2163. * p is forked by current.
  2164. *
  2165. * __sched_fork() is basic setup used by init_idle() too:
  2166. */
  2167. static void __sched_fork(struct task_struct *p)
  2168. {
  2169. p->se.exec_start = 0;
  2170. p->se.sum_exec_runtime = 0;
  2171. p->se.prev_sum_exec_runtime = 0;
  2172. p->se.nr_migrations = 0;
  2173. #ifdef CONFIG_SCHEDSTATS
  2174. memset(&p->se.statistics, 0, sizeof(p->se.statistics));
  2175. #endif
  2176. INIT_LIST_HEAD(&p->rt.run_list);
  2177. p->se.on_rq = 0;
  2178. INIT_LIST_HEAD(&p->se.group_node);
  2179. #ifdef CONFIG_PREEMPT_NOTIFIERS
  2180. INIT_HLIST_HEAD(&p->preempt_notifiers);
  2181. #endif
  2182. }
  2183. /*
  2184. * fork()/clone()-time setup:
  2185. */
  2186. void sched_fork(struct task_struct *p, int clone_flags)
  2187. {
  2188. int cpu = get_cpu();
  2189. __sched_fork(p);
  2190. /*
  2191. * We mark the process as running here. This guarantees that
  2192. * nobody will actually run it, and a signal or other external
  2193. * event cannot wake it up and insert it on the runqueue either.
  2194. */
  2195. p->state = TASK_RUNNING;
  2196. /*
  2197. * Revert to default priority/policy on fork if requested.
  2198. */
  2199. if (unlikely(p->sched_reset_on_fork)) {
  2200. if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) {
  2201. p->policy = SCHED_NORMAL;
  2202. p->normal_prio = p->static_prio;
  2203. }
  2204. if (PRIO_TO_NICE(p->static_prio) < 0) {
  2205. p->static_prio = NICE_TO_PRIO(0);
  2206. p->normal_prio = p->static_prio;
  2207. set_load_weight(p);
  2208. }
  2209. /*
  2210. * We don't need the reset flag anymore after the fork. It has
  2211. * fulfilled its duty:
  2212. */
  2213. p->sched_reset_on_fork = 0;
  2214. }
  2215. /*
  2216. * Make sure we do not leak PI boosting priority to the child.
  2217. */
  2218. p->prio = current->normal_prio;
  2219. if (!rt_prio(p->prio))
  2220. p->sched_class = &fair_sched_class;
  2221. if (p->sched_class->task_fork)
  2222. p->sched_class->task_fork(p);
  2223. /*
  2224. * The child is not yet in the pid-hash so no cgroup attach races,
  2225. * and the cgroup is pinned to this child due to cgroup_fork()
  2226. * is ran before sched_fork().
  2227. *
  2228. * Silence PROVE_RCU.
  2229. */
  2230. rcu_read_lock();
  2231. set_task_cpu(p, cpu);
  2232. rcu_read_unlock();
  2233. #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
  2234. if (likely(sched_info_on()))
  2235. memset(&p->sched_info, 0, sizeof(p->sched_info));
  2236. #endif
  2237. #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
  2238. p->oncpu = 0;
  2239. #endif
  2240. #ifdef CONFIG_PREEMPT
  2241. /* Want to start with kernel preemption disabled. */
  2242. task_thread_info(p)->preempt_count = 1;
  2243. #endif
  2244. #ifdef CONFIG_SMP
  2245. plist_node_init(&p->pushable_tasks, MAX_PRIO);
  2246. #endif
  2247. put_cpu();
  2248. }
  2249. /*
  2250. * wake_up_new_task - wake up a newly created task for the first time.
  2251. *
  2252. * This function will do some initial scheduler statistics housekeeping
  2253. * that must be done for every newly created context, then puts the task
  2254. * on the runqueue and wakes it.
  2255. */
  2256. void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
  2257. {
  2258. unsigned long flags;
  2259. struct rq *rq;
  2260. int cpu __maybe_unused = get_cpu();
  2261. #ifdef CONFIG_SMP
  2262. rq = task_rq_lock(p, &flags);
  2263. p->state = TASK_WAKING;
  2264. /*
  2265. * Fork balancing, do it here and not earlier because:
  2266. * - cpus_allowed can change in the fork path
  2267. * - any previously selected cpu might disappear through hotplug
  2268. *
  2269. * We set TASK_WAKING so that select_task_rq() can drop rq->lock
  2270. * without people poking at ->cpus_allowed.
  2271. */
  2272. cpu = select_task_rq(rq, p, SD_BALANCE_FORK, 0);
  2273. set_task_cpu(p, cpu);
  2274. p->state = TASK_RUNNING;
  2275. task_rq_unlock(rq, &flags);
  2276. #endif
  2277. rq = task_rq_lock(p, &flags);
  2278. activate_task(rq, p, 0);
  2279. trace_sched_wakeup_new(p, 1);
  2280. check_preempt_curr(rq, p, WF_FORK);
  2281. #ifdef CONFIG_SMP
  2282. if (p->sched_class->task_woken)
  2283. p->sched_class->task_woken(rq, p);
  2284. #endif
  2285. task_rq_unlock(rq, &flags);
  2286. put_cpu();
  2287. }
  2288. #ifdef CONFIG_PREEMPT_NOTIFIERS
  2289. /**
  2290. * preempt_notifier_register - tell me when current is being preempted & rescheduled
  2291. * @notifier: notifier struct to register
  2292. */
  2293. void preempt_notifier_register(struct preempt_notifier *notifier)
  2294. {
  2295. hlist_add_head(&notifier->link, &current->preempt_notifiers);
  2296. }
  2297. EXPORT_SYMBOL_GPL(preempt_notifier_register);
  2298. /**
  2299. * preempt_notifier_unregister - no longer interested in preemption notifications
  2300. * @notifier: notifier struct to unregister
  2301. *
  2302. * This is safe to call from within a preemption notifier.
  2303. */
  2304. void preempt_notifier_unregister(struct preempt_notifier *notifier)
  2305. {
  2306. hlist_del(&notifier->link);
  2307. }
  2308. EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
  2309. static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
  2310. {
  2311. struct preempt_notifier *notifier;
  2312. struct hlist_node *node;
  2313. hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
  2314. notifier->ops->sched_in(notifier, raw_smp_processor_id());
  2315. }
  2316. static void
  2317. fire_sched_out_preempt_notifiers(struct task_struct *curr,
  2318. struct task_struct *next)
  2319. {
  2320. struct preempt_notifier *notifier;
  2321. struct hlist_node *node;
  2322. hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
  2323. notifier->ops->sched_out(notifier, next);
  2324. }
  2325. #else /* !CONFIG_PREEMPT_NOTIFIERS */
  2326. static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
  2327. {
  2328. }
  2329. static void
  2330. fire_sched_out_preempt_notifiers(struct task_struct *curr,
  2331. struct task_struct *next)
  2332. {
  2333. }
  2334. #endif /* CONFIG_PREEMPT_NOTIFIERS */
  2335. /**
  2336. * prepare_task_switch - prepare to switch tasks
  2337. * @rq: the runqueue preparing to switch
  2338. * @prev: the current task that is being switched out
  2339. * @next: the task we are going to switch to.
  2340. *
  2341. * This is called with the rq lock held and interrupts off. It must
  2342. * be paired with a subsequent finish_task_switch after the context
  2343. * switch.
  2344. *
  2345. * prepare_task_switch sets up locking and calls architecture specific
  2346. * hooks.
  2347. */
  2348. static inline void
  2349. prepare_task_switch(struct rq *rq, struct task_struct *prev,
  2350. struct task_struct *next)
  2351. {
  2352. fire_sched_out_preempt_notifiers(prev, next);
  2353. prepare_lock_switch(rq, next);
  2354. prepare_arch_switch(next);
  2355. }
  2356. /**
  2357. * finish_task_switch - clean up after a task-switch
  2358. * @rq: runqueue associated with task-switch
  2359. * @prev: the thread we just switched away from.
  2360. *
  2361. * finish_task_switch must be called after the context switch, paired
  2362. * with a prepare_task_switch call before the context switch.
  2363. * finish_task_switch will reconcile locking set up by prepare_task_switch,
  2364. * and do any other architecture-specific cleanup actions.
  2365. *
  2366. * Note that we may have delayed dropping an mm in context_switch(). If
  2367. * so, we finish that here outside of the runqueue lock. (Doing it
  2368. * with the lock held can cause deadlocks; see schedule() for
  2369. * details.)
  2370. */
  2371. static void finish_task_switch(struct rq *rq, struct task_struct *prev)
  2372. __releases(rq->lock)
  2373. {
  2374. struct mm_struct *mm = rq->prev_mm;
  2375. long prev_state;
  2376. rq->prev_mm = NULL;
  2377. /*
  2378. * A task struct has one reference for the use as "current".
  2379. * If a task dies, then it sets TASK_DEAD in tsk->state and calls
  2380. * schedule one last time. The schedule call will never return, and
  2381. * the scheduled task must drop that reference.
  2382. * The test for TASK_DEAD must occur while the runqueue locks are
  2383. * still held, otherwise prev could be scheduled on another cpu, die
  2384. * there before we look at prev->state, and then the reference would
  2385. * be dropped twice.
  2386. * Manfred Spraul <manfred@colorfullife.com>
  2387. */
  2388. prev_state = prev->state;
  2389. finish_arch_switch(prev);
  2390. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  2391. local_irq_disable();
  2392. #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
  2393. perf_event_task_sched_in(current);
  2394. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  2395. local_irq_enable();
  2396. #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
  2397. finish_lock_switch(rq, prev);
  2398. fire_sched_in_preempt_notifiers(current);
  2399. if (mm)
  2400. mmdrop(mm);
  2401. if (unlikely(prev_state == TASK_DEAD)) {
  2402. /*
  2403. * Remove function-return probe instances associated with this
  2404. * task and put them back on the free list.
  2405. */
  2406. kprobe_flush_task(prev);
  2407. put_task_struct(prev);
  2408. }
  2409. }
  2410. #ifdef CONFIG_SMP
  2411. /* assumes rq->lock is held */
  2412. static inline void pre_schedule(struct rq *rq, struct task_struct *prev)
  2413. {
  2414. if (prev->sched_class->pre_schedule)
  2415. prev->sched_class->pre_schedule(rq, prev);
  2416. }
  2417. /* rq->lock is NOT held, but preemption is disabled */
  2418. static inline void post_schedule(struct rq *rq)
  2419. {
  2420. if (rq->post_schedule) {
  2421. unsigned long flags;
  2422. raw_spin_lock_irqsave(&rq->lock, flags);
  2423. if (rq->curr->sched_class->post_schedule)
  2424. rq->curr->sched_class->post_schedule(rq);
  2425. raw_spin_unlock_irqrestore(&rq->lock, flags);
  2426. rq->post_schedule = 0;
  2427. }
  2428. }
  2429. #else
  2430. static inline void pre_schedule(struct rq *rq, struct task_struct *p)
  2431. {
  2432. }
  2433. static inline void post_schedule(struct rq *rq)
  2434. {
  2435. }
  2436. #endif
  2437. /**
  2438. * schedule_tail - first thing a freshly forked thread must call.
  2439. * @prev: the thread we just switched away from.
  2440. */
  2441. asmlinkage void schedule_tail(struct task_struct *prev)
  2442. __releases(rq->lock)
  2443. {
  2444. struct rq *rq = this_rq();
  2445. finish_task_switch(rq, prev);
  2446. /*
  2447. * FIXME: do we need to worry about rq being invalidated by the
  2448. * task_switch?
  2449. */
  2450. post_schedule(rq);
  2451. #ifdef __ARCH_WANT_UNLOCKED_CTXSW
  2452. /* In this case, finish_task_switch does not reenable preemption */
  2453. preempt_enable();
  2454. #endif
  2455. if (current->set_child_tid)
  2456. put_user(task_pid_vnr(current), current->set_child_tid);
  2457. }
  2458. /*
  2459. * context_switch - switch to the new MM and the new
  2460. * thread's register state.
  2461. */
  2462. static inline void
  2463. context_switch(struct rq *rq, struct task_struct *prev,
  2464. struct task_struct *next)
  2465. {
  2466. struct mm_struct *mm, *oldmm;
  2467. prepare_task_switch(rq, prev, next);
  2468. trace_sched_switch(prev, next);
  2469. mm = next->mm;
  2470. oldmm = prev->active_mm;
  2471. /*
  2472. * For paravirt, this is coupled with an exit in switch_to to
  2473. * combine the page table reload and the switch backend into
  2474. * one hypercall.
  2475. */
  2476. arch_start_context_switch(prev);
  2477. if (!mm) {
  2478. next->active_mm = oldmm;
  2479. atomic_inc(&oldmm->mm_count);
  2480. enter_lazy_tlb(oldmm, next);
  2481. } else
  2482. switch_mm(oldmm, mm, next);
  2483. if (!prev->mm) {
  2484. prev->active_mm = NULL;
  2485. rq->prev_mm = oldmm;
  2486. }
  2487. /*
  2488. * Since the runqueue lock will be released by the next
  2489. * task (which is an invalid locking op but in the case
  2490. * of the scheduler it's an obvious special-case), so we
  2491. * do an early lockdep release here:
  2492. */
  2493. #ifndef __ARCH_WANT_UNLOCKED_CTXSW
  2494. spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
  2495. #endif
  2496. /* Here we just switch the register state and the stack. */
  2497. switch_to(prev, next, prev);
  2498. barrier();
  2499. /*
  2500. * this_rq must be evaluated again because prev may have moved
  2501. * CPUs since it called schedule(), thus the 'rq' on its stack
  2502. * frame will be invalid.
  2503. */
  2504. finish_task_switch(this_rq(), prev);
  2505. }
  2506. /*
  2507. * nr_running, nr_uninterruptible and nr_context_switches:
  2508. *
  2509. * externally visible scheduler statistics: current number of runnable
  2510. * threads, current number of uninterruptible-sleeping threads, total
  2511. * number of context switches performed since bootup.
  2512. */
  2513. unsigned long nr_running(void)
  2514. {
  2515. unsigned long i, sum = 0;
  2516. for_each_online_cpu(i)
  2517. sum += cpu_rq(i)->nr_running;
  2518. return sum;
  2519. }
  2520. unsigned long nr_uninterruptible(void)
  2521. {
  2522. unsigned long i, sum = 0;
  2523. for_each_possible_cpu(i)
  2524. sum += cpu_rq(i)->nr_uninterruptible;
  2525. /*
  2526. * Since we read the counters lockless, it might be slightly
  2527. * inaccurate. Do not allow it to go below zero though:
  2528. */
  2529. if (unlikely((long)sum < 0))
  2530. sum = 0;
  2531. return sum;
  2532. }
  2533. unsigned long long nr_context_switches(void)
  2534. {
  2535. int i;
  2536. unsigned long long sum = 0;
  2537. for_each_possible_cpu(i)
  2538. sum += cpu_rq(i)->nr_switches;
  2539. return sum;
  2540. }
  2541. unsigned long nr_iowait(void)
  2542. {
  2543. unsigned long i, sum = 0;
  2544. for_each_possible_cpu(i)
  2545. sum += atomic_read(&cpu_rq(i)->nr_iowait);
  2546. return sum;
  2547. }
  2548. unsigned long nr_iowait_cpu(int cpu)
  2549. {
  2550. struct rq *this = cpu_rq(cpu);
  2551. return atomic_read(&this->nr_iowait);
  2552. }
  2553. unsigned long this_cpu_load(void)
  2554. {
  2555. struct rq *this = this_rq();
  2556. return this->cpu_load[0];
  2557. }
  2558. /* Variables and functions for calc_load */
  2559. static atomic_long_t calc_load_tasks;
  2560. static unsigned long calc_load_update;
  2561. unsigned long avenrun[3];
  2562. EXPORT_SYMBOL(avenrun);
  2563. static long calc_load_fold_active(struct rq *this_rq)
  2564. {
  2565. long nr_active, delta = 0;
  2566. nr_active = this_rq->nr_running;
  2567. nr_active += (long) this_rq->nr_uninterruptible;
  2568. if (nr_active != this_rq->calc_load_active) {
  2569. delta = nr_active - this_rq->calc_load_active;
  2570. this_rq->calc_load_active = nr_active;
  2571. }
  2572. return delta;
  2573. }
  2574. static unsigned long
  2575. calc_load(unsigned long load, unsigned long exp, unsigned long active)
  2576. {
  2577. load *= exp;
  2578. load += active * (FIXED_1 - exp);
  2579. load += 1UL << (FSHIFT - 1);
  2580. return load >> FSHIFT;
  2581. }
  2582. #ifdef CONFIG_NO_HZ
  2583. /*
  2584. * For NO_HZ we delay the active fold to the next LOAD_FREQ update.
  2585. *
  2586. * When making the ILB scale, we should try to pull this in as well.
  2587. */
  2588. static atomic_long_t calc_load_tasks_idle;
  2589. static void calc_load_account_idle(struct rq *this_rq)
  2590. {
  2591. long delta;
  2592. delta = calc_load_fold_active(this_rq);
  2593. if (delta)
  2594. atomic_long_add(delta, &calc_load_tasks_idle);
  2595. }
  2596. static long calc_load_fold_idle(void)
  2597. {
  2598. long delta = 0;
  2599. /*
  2600. * Its got a race, we don't care...
  2601. */
  2602. if (atomic_long_read(&calc_load_tasks_idle))
  2603. delta = atomic_long_xchg(&calc_load_tasks_idle, 0);
  2604. return delta;
  2605. }
  2606. /**
  2607. * fixed_power_int - compute: x^n, in O(log n) time
  2608. *
  2609. * @x: base of the power
  2610. * @frac_bits: fractional bits of @x
  2611. * @n: power to raise @x to.
  2612. *
  2613. * By exploiting the relation between the definition of the natural power
  2614. * function: x^n := x*x*...*x (x multiplied by itself for n times), and
  2615. * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i,
  2616. * (where: n_i \elem {0, 1}, the binary vector representing n),
  2617. * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is
  2618. * of course trivially computable in O(log_2 n), the length of our binary
  2619. * vector.
  2620. */
  2621. static unsigned long
  2622. fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n)
  2623. {
  2624. unsigned long result = 1UL << frac_bits;
  2625. if (n) for (;;) {
  2626. if (n & 1) {
  2627. result *= x;
  2628. result += 1UL << (frac_bits - 1);
  2629. result >>= frac_bits;
  2630. }
  2631. n >>= 1;
  2632. if (!n)
  2633. break;
  2634. x *= x;
  2635. x += 1UL << (frac_bits - 1);
  2636. x >>= frac_bits;
  2637. }
  2638. return result;
  2639. }
  2640. /*
  2641. * a1 = a0 * e + a * (1 - e)
  2642. *
  2643. * a2 = a1 * e + a * (1 - e)
  2644. * = (a0 * e + a * (1 - e)) * e + a * (1 - e)
  2645. * = a0 * e^2 + a * (1 - e) * (1 + e)
  2646. *
  2647. * a3 = a2 * e + a * (1 - e)
  2648. * = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e)
  2649. * = a0 * e^3 + a * (1 - e) * (1 + e + e^2)
  2650. *
  2651. * ...
  2652. *
  2653. * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1]
  2654. * = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e)
  2655. * = a0 * e^n + a * (1 - e^n)
  2656. *
  2657. * [1] application of the geometric series:
  2658. *
  2659. * n 1 - x^(n+1)
  2660. * S_n := \Sum x^i = -------------
  2661. * i=0 1 - x
  2662. */
  2663. static unsigned long
  2664. calc_load_n(unsigned long load, unsigned long exp,
  2665. unsigned long active, unsigned int n)
  2666. {
  2667. return calc_load(load, fixed_power_int(exp, FSHIFT, n), active);
  2668. }
  2669. /*
  2670. * NO_HZ can leave us missing all per-cpu ticks calling
  2671. * calc_load_account_active(), but since an idle CPU folds its delta into
  2672. * calc_load_tasks_idle per calc_load_account_idle(), all we need to do is fold
  2673. * in the pending idle delta if our idle period crossed a load cycle boundary.
  2674. *
  2675. * Once we've updated the global active value, we need to apply the exponential
  2676. * weights adjusted to the number of cycles missed.
  2677. */
  2678. static void calc_global_nohz(unsigned long ticks)
  2679. {
  2680. long delta, active, n;
  2681. if (time_before(jiffies, calc_load_update))
  2682. return;
  2683. /*
  2684. * If we crossed a calc_load_update boundary, make sure to fold
  2685. * any pending idle changes, the respective CPUs might have
  2686. * missed the tick driven calc_load_account_active() update
  2687. * due to NO_HZ.
  2688. */
  2689. delta = calc_load_fold_idle();
  2690. if (delta)
  2691. atomic_long_add(delta, &calc_load_tasks);
  2692. /*
  2693. * If we were idle for multiple load cycles, apply them.
  2694. */
  2695. if (ticks >= LOAD_FREQ) {
  2696. n = ticks / LOAD_FREQ;
  2697. active = atomic_long_read(&calc_load_tasks);
  2698. active = active > 0 ? active * FIXED_1 : 0;
  2699. avenrun[0] = calc_load_n(avenrun[0], EXP_1, active, n);
  2700. avenrun[1] = calc_load_n(avenrun[1], EXP_5, active, n);
  2701. avenrun[2] = calc_load_n(avenrun[2], EXP_15, active, n);
  2702. calc_load_update += n * LOAD_FREQ;
  2703. }
  2704. /*
  2705. * Its possible the remainder of the above division also crosses
  2706. * a LOAD_FREQ period, the regular check in calc_global_load()
  2707. * which comes after this will take care of that.
  2708. *
  2709. * Consider us being 11 ticks before a cycle completion, and us
  2710. * sleeping for 4*LOAD_FREQ + 22 ticks, then the above code will
  2711. * age us 4 cycles, and the test in calc_global_load() will
  2712. * pick up the final one.
  2713. */
  2714. }
  2715. #else
  2716. static void calc_load_account_idle(struct rq *this_rq)
  2717. {
  2718. }
  2719. static inline long calc_load_fold_idle(void)
  2720. {
  2721. return 0;
  2722. }
  2723. static void calc_global_nohz(unsigned long ticks)
  2724. {
  2725. }
  2726. #endif
  2727. /**
  2728. * get_avenrun - get the load average array
  2729. * @loads: pointer to dest load array
  2730. * @offset: offset to add
  2731. * @shift: shift count to shift the result left
  2732. *
  2733. * These values are estimates at best, so no need for locking.
  2734. */
  2735. void get_avenrun(unsigned long *loads, unsigned long offset, int shift)
  2736. {
  2737. loads[0] = (avenrun[0] + offset) << shift;
  2738. loads[1] = (avenrun[1] + offset) << shift;
  2739. loads[2] = (avenrun[2] + offset) << shift;
  2740. }
  2741. /*
  2742. * calc_load - update the avenrun load estimates 10 ticks after the
  2743. * CPUs have updated calc_load_tasks.
  2744. */
  2745. void calc_global_load(unsigned long ticks)
  2746. {
  2747. long active;
  2748. calc_global_nohz(ticks);
  2749. if (time_before(jiffies, calc_load_update + 10))
  2750. return;
  2751. active = atomic_long_read(&calc_load_tasks);
  2752. active = active > 0 ? active * FIXED_1 : 0;
  2753. avenrun[0] = calc_load(avenrun[0], EXP_1, active);
  2754. avenrun[1] = calc_load(avenrun[1], EXP_5, active);
  2755. avenrun[2] = calc_load(avenrun[2], EXP_15, active);
  2756. calc_load_update += LOAD_FREQ;
  2757. }
  2758. /*
  2759. * Called from update_cpu_load() to periodically update this CPU's
  2760. * active count.
  2761. */
  2762. static void calc_load_account_active(struct rq *this_rq)
  2763. {
  2764. long delta;
  2765. if (time_before(jiffies, this_rq->calc_load_update))
  2766. return;
  2767. delta = calc_load_fold_active(this_rq);
  2768. delta += calc_load_fold_idle();
  2769. if (delta)
  2770. atomic_long_add(delta, &calc_load_tasks);
  2771. this_rq->calc_load_update += LOAD_FREQ;
  2772. }
  2773. /*
  2774. * The exact cpuload at various idx values, calculated at every tick would be
  2775. * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load
  2776. *
  2777. * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called
  2778. * on nth tick when cpu may be busy, then we have:
  2779. * load = ((2^idx - 1) / 2^idx)^(n-1) * load
  2780. * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load
  2781. *
  2782. * decay_load_missed() below does efficient calculation of
  2783. * load = ((2^idx - 1) / 2^idx)^(n-1) * load
  2784. * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load
  2785. *
  2786. * The calculation is approximated on a 128 point scale.
  2787. * degrade_zero_ticks is the number of ticks after which load at any
  2788. * particular idx is approximated to be zero.
  2789. * degrade_factor is a precomputed table, a row for each load idx.
  2790. * Each column corresponds to degradation factor for a power of two ticks,
  2791. * based on 128 point scale.
  2792. * Example:
  2793. * row 2, col 3 (=12) says that the degradation at load idx 2 after
  2794. * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8).
  2795. *
  2796. * With this power of 2 load factors, we can degrade the load n times
  2797. * by looking at 1 bits in n and doing as many mult/shift instead of
  2798. * n mult/shifts needed by the exact degradation.
  2799. */
  2800. #define DEGRADE_SHIFT 7
  2801. static const unsigned char
  2802. degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128};
  2803. static const unsigned char
  2804. degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = {
  2805. {0, 0, 0, 0, 0, 0, 0, 0},
  2806. {64, 32, 8, 0, 0, 0, 0, 0},
  2807. {96, 72, 40, 12, 1, 0, 0},
  2808. {112, 98, 75, 43, 15, 1, 0},
  2809. {120, 112, 98, 76, 45, 16, 2} };
  2810. /*
  2811. * Update cpu_load for any missed ticks, due to tickless idle. The backlog
  2812. * would be when CPU is idle and so we just decay the old load without
  2813. * adding any new load.
  2814. */
  2815. static unsigned long
  2816. decay_load_missed(unsigned long load, unsigned long missed_updates, int idx)
  2817. {
  2818. int j = 0;
  2819. if (!missed_updates)
  2820. return load;
  2821. if (missed_updates >= degrade_zero_ticks[idx])
  2822. return 0;
  2823. if (idx == 1)
  2824. return load >> missed_updates;
  2825. while (missed_updates) {
  2826. if (missed_updates % 2)
  2827. load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT;
  2828. missed_updates >>= 1;
  2829. j++;
  2830. }
  2831. return load;
  2832. }
  2833. /*
  2834. * Update rq->cpu_load[] statistics. This function is usually called every
  2835. * scheduler tick (TICK_NSEC). With tickless idle this will not be called
  2836. * every tick. We fix it up based on jiffies.
  2837. */
  2838. static void update_cpu_load(struct rq *this_rq)
  2839. {
  2840. unsigned long this_load = this_rq->load.weight;
  2841. unsigned long curr_jiffies = jiffies;
  2842. unsigned long pending_updates;
  2843. int i, scale;
  2844. this_rq->nr_load_updates++;
  2845. /* Avoid repeated calls on same jiffy, when moving in and out of idle */
  2846. if (curr_jiffies == this_rq->last_load_update_tick)
  2847. return;
  2848. pending_updates = curr_jiffies - this_rq->last_load_update_tick;
  2849. this_rq->last_load_update_tick = curr_jiffies;
  2850. /* Update our load: */
  2851. this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */
  2852. for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
  2853. unsigned long old_load, new_load;
  2854. /* scale is effectively 1 << i now, and >> i divides by scale */
  2855. old_load = this_rq->cpu_load[i];
  2856. old_load = decay_load_missed(old_load, pending_updates - 1, i);
  2857. new_load = this_load;
  2858. /*
  2859. * Round up the averaging division if load is increasing. This
  2860. * prevents us from getting stuck on 9 if the load is 10, for
  2861. * example.
  2862. */
  2863. if (new_load > old_load)
  2864. new_load += scale - 1;
  2865. this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i;
  2866. }
  2867. sched_avg_update(this_rq);
  2868. }
  2869. static void update_cpu_load_active(struct rq *this_rq)
  2870. {
  2871. update_cpu_load(this_rq);
  2872. calc_load_account_active(this_rq);
  2873. }
  2874. #ifdef CONFIG_SMP
  2875. /*
  2876. * sched_exec - execve() is a valuable balancing opportunity, because at
  2877. * this point the task has the smallest effective memory and cache footprint.
  2878. */
  2879. void sched_exec(void)
  2880. {
  2881. struct task_struct *p = current;
  2882. unsigned long flags;
  2883. struct rq *rq;
  2884. int dest_cpu;
  2885. rq = task_rq_lock(p, &flags);
  2886. dest_cpu = p->sched_class->select_task_rq(rq, p, SD_BALANCE_EXEC, 0);
  2887. if (dest_cpu == smp_processor_id())
  2888. goto unlock;
  2889. /*
  2890. * select_task_rq() can race against ->cpus_allowed
  2891. */
  2892. if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed) &&
  2893. likely(cpu_active(dest_cpu)) && migrate_task(p, rq)) {
  2894. struct migration_arg arg = { p, dest_cpu };
  2895. task_rq_unlock(rq, &flags);
  2896. stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
  2897. return;
  2898. }
  2899. unlock:
  2900. task_rq_unlock(rq, &flags);
  2901. }
  2902. #endif
  2903. DEFINE_PER_CPU(struct kernel_stat, kstat);
  2904. EXPORT_PER_CPU_SYMBOL(kstat);
  2905. /*
  2906. * Return any ns on the sched_clock that have not yet been accounted in
  2907. * @p in case that task is currently running.
  2908. *
  2909. * Called with task_rq_lock() held on @rq.
  2910. */
  2911. static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
  2912. {
  2913. u64 ns = 0;
  2914. if (task_current(rq, p)) {
  2915. update_rq_clock(rq);
  2916. ns = rq->clock_task - p->se.exec_start;
  2917. if ((s64)ns < 0)
  2918. ns = 0;
  2919. }
  2920. return ns;
  2921. }
  2922. unsigned long long task_delta_exec(struct task_struct *p)
  2923. {
  2924. unsigned long flags;
  2925. struct rq *rq;
  2926. u64 ns = 0;
  2927. rq = task_rq_lock(p, &flags);
  2928. ns = do_task_delta_exec(p, rq);
  2929. task_rq_unlock(rq, &flags);
  2930. return ns;
  2931. }
  2932. /*
  2933. * Return accounted runtime for the task.
  2934. * In case the task is currently running, return the runtime plus current's
  2935. * pending runtime that have not been accounted yet.
  2936. */
  2937. unsigned long long task_sched_runtime(struct task_struct *p)
  2938. {
  2939. unsigned long flags;
  2940. struct rq *rq;
  2941. u64 ns = 0;
  2942. rq = task_rq_lock(p, &flags);
  2943. ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
  2944. task_rq_unlock(rq, &flags);
  2945. return ns;
  2946. }
  2947. /*
  2948. * Return sum_exec_runtime for the thread group.
  2949. * In case the task is currently running, return the sum plus current's
  2950. * pending runtime that have not been accounted yet.
  2951. *
  2952. * Note that the thread group might have other running tasks as well,
  2953. * so the return value not includes other pending runtime that other
  2954. * running tasks might have.
  2955. */
  2956. unsigned long long thread_group_sched_runtime(struct task_struct *p)
  2957. {
  2958. struct task_cputime totals;
  2959. unsigned long flags;
  2960. struct rq *rq;
  2961. u64 ns;
  2962. rq = task_rq_lock(p, &flags);
  2963. thread_group_cputime(p, &totals);
  2964. ns = totals.sum_exec_runtime + do_task_delta_exec(p, rq);
  2965. task_rq_unlock(rq, &flags);
  2966. return ns;
  2967. }
  2968. /*
  2969. * Account user cpu time to a process.
  2970. * @p: the process that the cpu time gets accounted to
  2971. * @cputime: the cpu time spent in user space since the last update
  2972. * @cputime_scaled: cputime scaled by cpu frequency
  2973. */
  2974. void account_user_time(struct task_struct *p, cputime_t cputime,
  2975. cputime_t cputime_scaled)
  2976. {
  2977. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  2978. cputime64_t tmp;
  2979. /* Add user time to process. */
  2980. p->utime = cputime_add(p->utime, cputime);
  2981. p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
  2982. account_group_user_time(p, cputime);
  2983. /* Add user time to cpustat. */
  2984. tmp = cputime_to_cputime64(cputime);
  2985. if (TASK_NICE(p) > 0)
  2986. cpustat->nice = cputime64_add(cpustat->nice, tmp);
  2987. else
  2988. cpustat->user = cputime64_add(cpustat->user, tmp);
  2989. cpuacct_update_stats(p, CPUACCT_STAT_USER, cputime);
  2990. /* Account for user time used */
  2991. acct_update_integrals(p);
  2992. }
  2993. /*
  2994. * Account guest cpu time to a process.
  2995. * @p: the process that the cpu time gets accounted to
  2996. * @cputime: the cpu time spent in virtual machine since the last update
  2997. * @cputime_scaled: cputime scaled by cpu frequency
  2998. */
  2999. static void account_guest_time(struct task_struct *p, cputime_t cputime,
  3000. cputime_t cputime_scaled)
  3001. {
  3002. cputime64_t tmp;
  3003. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3004. tmp = cputime_to_cputime64(cputime);
  3005. /* Add guest time to process. */
  3006. p->utime = cputime_add(p->utime, cputime);
  3007. p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
  3008. account_group_user_time(p, cputime);
  3009. p->gtime = cputime_add(p->gtime, cputime);
  3010. /* Add guest time to cpustat. */
  3011. if (TASK_NICE(p) > 0) {
  3012. cpustat->nice = cputime64_add(cpustat->nice, tmp);
  3013. cpustat->guest_nice = cputime64_add(cpustat->guest_nice, tmp);
  3014. } else {
  3015. cpustat->user = cputime64_add(cpustat->user, tmp);
  3016. cpustat->guest = cputime64_add(cpustat->guest, tmp);
  3017. }
  3018. }
  3019. /*
  3020. * Account system cpu time to a process.
  3021. * @p: the process that the cpu time gets accounted to
  3022. * @hardirq_offset: the offset to subtract from hardirq_count()
  3023. * @cputime: the cpu time spent in kernel space since the last update
  3024. * @cputime_scaled: cputime scaled by cpu frequency
  3025. */
  3026. void account_system_time(struct task_struct *p, int hardirq_offset,
  3027. cputime_t cputime, cputime_t cputime_scaled)
  3028. {
  3029. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3030. cputime64_t tmp;
  3031. if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
  3032. account_guest_time(p, cputime, cputime_scaled);
  3033. return;
  3034. }
  3035. /* Add system time to process. */
  3036. p->stime = cputime_add(p->stime, cputime);
  3037. p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
  3038. account_group_system_time(p, cputime);
  3039. /* Add system time to cpustat. */
  3040. tmp = cputime_to_cputime64(cputime);
  3041. if (hardirq_count() - hardirq_offset)
  3042. cpustat->irq = cputime64_add(cpustat->irq, tmp);
  3043. else if (in_serving_softirq())
  3044. cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
  3045. else
  3046. cpustat->system = cputime64_add(cpustat->system, tmp);
  3047. cpuacct_update_stats(p, CPUACCT_STAT_SYSTEM, cputime);
  3048. /* Account for system time used */
  3049. acct_update_integrals(p);
  3050. }
  3051. /*
  3052. * Account for involuntary wait time.
  3053. * @steal: the cpu time spent in involuntary wait
  3054. */
  3055. void account_steal_time(cputime_t cputime)
  3056. {
  3057. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3058. cputime64_t cputime64 = cputime_to_cputime64(cputime);
  3059. cpustat->steal = cputime64_add(cpustat->steal, cputime64);
  3060. }
  3061. /*
  3062. * Account for idle time.
  3063. * @cputime: the cpu time spent in idle wait
  3064. */
  3065. void account_idle_time(cputime_t cputime)
  3066. {
  3067. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3068. cputime64_t cputime64 = cputime_to_cputime64(cputime);
  3069. struct rq *rq = this_rq();
  3070. if (atomic_read(&rq->nr_iowait) > 0)
  3071. cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
  3072. else
  3073. cpustat->idle = cputime64_add(cpustat->idle, cputime64);
  3074. }
  3075. #ifndef CONFIG_VIRT_CPU_ACCOUNTING
  3076. /*
  3077. * Account a single tick of cpu time.
  3078. * @p: the process that the cpu time gets accounted to
  3079. * @user_tick: indicates if the tick is a user or a system tick
  3080. */
  3081. void account_process_tick(struct task_struct *p, int user_tick)
  3082. {
  3083. cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
  3084. struct rq *rq = this_rq();
  3085. if (user_tick)
  3086. account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
  3087. else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET))
  3088. account_system_time(p, HARDIRQ_OFFSET, cputime_one_jiffy,
  3089. one_jiffy_scaled);
  3090. else
  3091. account_idle_time(cputime_one_jiffy);
  3092. }
  3093. /*
  3094. * Account multiple ticks of steal time.
  3095. * @p: the process from which the cpu time has been stolen
  3096. * @ticks: number of stolen ticks
  3097. */
  3098. void account_steal_ticks(unsigned long ticks)
  3099. {
  3100. account_steal_time(jiffies_to_cputime(ticks));
  3101. }
  3102. /*
  3103. * Account multiple ticks of idle time.
  3104. * @ticks: number of stolen ticks
  3105. */
  3106. void account_idle_ticks(unsigned long ticks)
  3107. {
  3108. account_idle_time(jiffies_to_cputime(ticks));
  3109. }
  3110. #endif
  3111. /*
  3112. * Use precise platform statistics if available:
  3113. */
  3114. #ifdef CONFIG_VIRT_CPU_ACCOUNTING
  3115. void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
  3116. {
  3117. *ut = p->utime;
  3118. *st = p->stime;
  3119. }
  3120. void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
  3121. {
  3122. struct task_cputime cputime;
  3123. thread_group_cputime(p, &cputime);
  3124. *ut = cputime.utime;
  3125. *st = cputime.stime;
  3126. }
  3127. #else
  3128. #ifndef nsecs_to_cputime
  3129. # define nsecs_to_cputime(__nsecs) nsecs_to_jiffies(__nsecs)
  3130. #endif
  3131. void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
  3132. {
  3133. cputime_t rtime, utime = p->utime, total = cputime_add(utime, p->stime);
  3134. /*
  3135. * Use CFS's precise accounting:
  3136. */
  3137. rtime = nsecs_to_cputime(p->se.sum_exec_runtime);
  3138. if (total) {
  3139. u64 temp = rtime;
  3140. temp *= utime;
  3141. do_div(temp, total);
  3142. utime = (cputime_t)temp;
  3143. } else
  3144. utime = rtime;
  3145. /*
  3146. * Compare with previous values, to keep monotonicity:
  3147. */
  3148. p->prev_utime = max(p->prev_utime, utime);
  3149. p->prev_stime = max(p->prev_stime, cputime_sub(rtime, p->prev_utime));
  3150. *ut = p->prev_utime;
  3151. *st = p->prev_stime;
  3152. }
  3153. /*
  3154. * Must be called with siglock held.
  3155. */
  3156. void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
  3157. {
  3158. struct signal_struct *sig = p->signal;
  3159. struct task_cputime cputime;
  3160. cputime_t rtime, utime, total;
  3161. thread_group_cputime(p, &cputime);
  3162. total = cputime_add(cputime.utime, cputime.stime);
  3163. rtime = nsecs_to_cputime(cputime.sum_exec_runtime);
  3164. if (total) {
  3165. u64 temp = rtime;
  3166. temp *= cputime.utime;
  3167. do_div(temp, total);
  3168. utime = (cputime_t)temp;
  3169. } else
  3170. utime = rtime;
  3171. sig->prev_utime = max(sig->prev_utime, utime);
  3172. sig->prev_stime = max(sig->prev_stime,
  3173. cputime_sub(rtime, sig->prev_utime));
  3174. *ut = sig->prev_utime;
  3175. *st = sig->prev_stime;
  3176. }
  3177. #endif
  3178. /*
  3179. * This function gets called by the timer code, with HZ frequency.
  3180. * We call it with interrupts disabled.
  3181. *
  3182. * It also gets called by the fork code, when changing the parent's
  3183. * timeslices.
  3184. */
  3185. void scheduler_tick(void)
  3186. {
  3187. int cpu = smp_processor_id();
  3188. struct rq *rq = cpu_rq(cpu);
  3189. struct task_struct *curr = rq->curr;
  3190. sched_clock_tick();
  3191. raw_spin_lock(&rq->lock);
  3192. update_rq_clock(rq);
  3193. update_cpu_load_active(rq);
  3194. curr->sched_class->task_tick(rq, curr, 0);
  3195. raw_spin_unlock(&rq->lock);
  3196. perf_event_task_tick();
  3197. #ifdef CONFIG_SMP
  3198. rq->idle_at_tick = idle_cpu(cpu);
  3199. trigger_load_balance(rq, cpu);
  3200. #endif
  3201. }
  3202. notrace unsigned long get_parent_ip(unsigned long addr)
  3203. {
  3204. if (in_lock_functions(addr)) {
  3205. addr = CALLER_ADDR2;
  3206. if (in_lock_functions(addr))
  3207. addr = CALLER_ADDR3;
  3208. }
  3209. return addr;
  3210. }
  3211. #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
  3212. defined(CONFIG_PREEMPT_TRACER))
  3213. void __kprobes add_preempt_count(int val)
  3214. {
  3215. #ifdef CONFIG_DEBUG_PREEMPT
  3216. /*
  3217. * Underflow?
  3218. */
  3219. if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
  3220. return;
  3221. #endif
  3222. preempt_count() += val;
  3223. #ifdef CONFIG_DEBUG_PREEMPT
  3224. /*
  3225. * Spinlock count overflowing soon?
  3226. */
  3227. DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
  3228. PREEMPT_MASK - 10);
  3229. #endif
  3230. if (preempt_count() == val)
  3231. trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
  3232. }
  3233. EXPORT_SYMBOL(add_preempt_count);
  3234. void __kprobes sub_preempt_count(int val)
  3235. {
  3236. #ifdef CONFIG_DEBUG_PREEMPT
  3237. /*
  3238. * Underflow?
  3239. */
  3240. if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
  3241. return;
  3242. /*
  3243. * Is the spinlock portion underflowing?
  3244. */
  3245. if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
  3246. !(preempt_count() & PREEMPT_MASK)))
  3247. return;
  3248. #endif
  3249. if (preempt_count() == val)
  3250. trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
  3251. preempt_count() -= val;
  3252. }
  3253. EXPORT_SYMBOL(sub_preempt_count);
  3254. #endif
  3255. /*
  3256. * Print scheduling while atomic bug:
  3257. */
  3258. static noinline void __schedule_bug(struct task_struct *prev)
  3259. {
  3260. struct pt_regs *regs = get_irq_regs();
  3261. printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
  3262. prev->comm, prev->pid, preempt_count());
  3263. debug_show_held_locks(prev);
  3264. print_modules();
  3265. if (irqs_disabled())
  3266. print_irqtrace_events(prev);
  3267. if (regs)
  3268. show_regs(regs);
  3269. else
  3270. dump_stack();
  3271. }
  3272. /*
  3273. * Various schedule()-time debugging checks and statistics:
  3274. */
  3275. static inline void schedule_debug(struct task_struct *prev)
  3276. {
  3277. /*
  3278. * Test if we are atomic. Since do_exit() needs to call into
  3279. * schedule() atomically, we ignore that path for now.
  3280. * Otherwise, whine if we are scheduling when we should not be.
  3281. */
  3282. if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
  3283. __schedule_bug(prev);
  3284. profile_hit(SCHED_PROFILING, __builtin_return_address(0));
  3285. schedstat_inc(this_rq(), sched_count);
  3286. #ifdef CONFIG_SCHEDSTATS
  3287. if (unlikely(prev->lock_depth >= 0)) {
  3288. schedstat_inc(this_rq(), bkl_count);
  3289. schedstat_inc(prev, sched_info.bkl_count);
  3290. }
  3291. #endif
  3292. }
  3293. static void put_prev_task(struct rq *rq, struct task_struct *prev)
  3294. {
  3295. if (prev->se.on_rq)
  3296. update_rq_clock(rq);
  3297. prev->sched_class->put_prev_task(rq, prev);
  3298. }
  3299. /*
  3300. * Pick up the highest-prio task:
  3301. */
  3302. static inline struct task_struct *
  3303. pick_next_task(struct rq *rq)
  3304. {
  3305. const struct sched_class *class;
  3306. struct task_struct *p;
  3307. /*
  3308. * Optimization: we know that if all tasks are in
  3309. * the fair class we can call that function directly:
  3310. */
  3311. if (likely(rq->nr_running == rq->cfs.nr_running)) {
  3312. p = fair_sched_class.pick_next_task(rq);
  3313. if (likely(p))
  3314. return p;
  3315. }
  3316. for_each_class(class) {
  3317. p = class->pick_next_task(rq);
  3318. if (p)
  3319. return p;
  3320. }
  3321. BUG(); /* the idle class will always have a runnable task */
  3322. }
  3323. /*
  3324. * schedule() is the main scheduler function.
  3325. */
  3326. asmlinkage void __sched schedule(void)
  3327. {
  3328. struct task_struct *prev, *next;
  3329. unsigned long *switch_count;
  3330. struct rq *rq;
  3331. int cpu;
  3332. need_resched:
  3333. preempt_disable();
  3334. cpu = smp_processor_id();
  3335. rq = cpu_rq(cpu);
  3336. rcu_note_context_switch(cpu);
  3337. prev = rq->curr;
  3338. release_kernel_lock(prev);
  3339. need_resched_nonpreemptible:
  3340. schedule_debug(prev);
  3341. if (sched_feat(HRTICK))
  3342. hrtick_clear(rq);
  3343. raw_spin_lock_irq(&rq->lock);
  3344. switch_count = &prev->nivcsw;
  3345. if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
  3346. if (unlikely(signal_pending_state(prev->state, prev))) {
  3347. prev->state = TASK_RUNNING;
  3348. } else {
  3349. /*
  3350. * If a worker is going to sleep, notify and
  3351. * ask workqueue whether it wants to wake up a
  3352. * task to maintain concurrency. If so, wake
  3353. * up the task.
  3354. */
  3355. if (prev->flags & PF_WQ_WORKER) {
  3356. struct task_struct *to_wakeup;
  3357. to_wakeup = wq_worker_sleeping(prev, cpu);
  3358. if (to_wakeup)
  3359. try_to_wake_up_local(to_wakeup);
  3360. }
  3361. deactivate_task(rq, prev, DEQUEUE_SLEEP);
  3362. }
  3363. switch_count = &prev->nvcsw;
  3364. }
  3365. pre_schedule(rq, prev);
  3366. if (unlikely(!rq->nr_running))
  3367. idle_balance(cpu, rq);
  3368. put_prev_task(rq, prev);
  3369. next = pick_next_task(rq);
  3370. clear_tsk_need_resched(prev);
  3371. rq->skip_clock_update = 0;
  3372. if (likely(prev != next)) {
  3373. sched_info_switch(prev, next);
  3374. perf_event_task_sched_out(prev, next);
  3375. rq->nr_switches++;
  3376. rq->curr = next;
  3377. ++*switch_count;
  3378. context_switch(rq, prev, next); /* unlocks the rq */
  3379. /*
  3380. * The context switch have flipped the stack from under us
  3381. * and restored the local variables which were saved when
  3382. * this task called schedule() in the past. prev == current
  3383. * is still correct, but it can be moved to another cpu/rq.
  3384. */
  3385. cpu = smp_processor_id();
  3386. rq = cpu_rq(cpu);
  3387. } else
  3388. raw_spin_unlock_irq(&rq->lock);
  3389. post_schedule(rq);
  3390. if (unlikely(reacquire_kernel_lock(prev)))
  3391. goto need_resched_nonpreemptible;
  3392. preempt_enable_no_resched();
  3393. if (need_resched())
  3394. goto need_resched;
  3395. }
  3396. EXPORT_SYMBOL(schedule);
  3397. #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
  3398. /*
  3399. * Look out! "owner" is an entirely speculative pointer
  3400. * access and not reliable.
  3401. */
  3402. int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner)
  3403. {
  3404. unsigned int cpu;
  3405. struct rq *rq;
  3406. if (!sched_feat(OWNER_SPIN))
  3407. return 0;
  3408. #ifdef CONFIG_DEBUG_PAGEALLOC
  3409. /*
  3410. * Need to access the cpu field knowing that
  3411. * DEBUG_PAGEALLOC could have unmapped it if
  3412. * the mutex owner just released it and exited.
  3413. */
  3414. if (probe_kernel_address(&owner->cpu, cpu))
  3415. return 0;
  3416. #else
  3417. cpu = owner->cpu;
  3418. #endif
  3419. /*
  3420. * Even if the access succeeded (likely case),
  3421. * the cpu field may no longer be valid.
  3422. */
  3423. if (cpu >= nr_cpumask_bits)
  3424. return 0;
  3425. /*
  3426. * We need to validate that we can do a
  3427. * get_cpu() and that we have the percpu area.
  3428. */
  3429. if (!cpu_online(cpu))
  3430. return 0;
  3431. rq = cpu_rq(cpu);
  3432. for (;;) {
  3433. /*
  3434. * Owner changed, break to re-assess state.
  3435. */
  3436. if (lock->owner != owner) {
  3437. /*
  3438. * If the lock has switched to a different owner,
  3439. * we likely have heavy contention. Return 0 to quit
  3440. * optimistic spinning and not contend further:
  3441. */
  3442. if (lock->owner)
  3443. return 0;
  3444. break;
  3445. }
  3446. /*
  3447. * Is that owner really running on that cpu?
  3448. */
  3449. if (task_thread_info(rq->curr) != owner || need_resched())
  3450. return 0;
  3451. arch_mutex_cpu_relax();
  3452. }
  3453. return 1;
  3454. }
  3455. #endif
  3456. #ifdef CONFIG_PREEMPT
  3457. /*
  3458. * this is the entry point to schedule() from in-kernel preemption
  3459. * off of preempt_enable. Kernel preemptions off return from interrupt
  3460. * occur there and call schedule directly.
  3461. */
  3462. asmlinkage void __sched notrace preempt_schedule(void)
  3463. {
  3464. struct thread_info *ti = current_thread_info();
  3465. /*
  3466. * If there is a non-zero preempt_count or interrupts are disabled,
  3467. * we do not want to preempt the current task. Just return..
  3468. */
  3469. if (likely(ti->preempt_count || irqs_disabled()))
  3470. return;
  3471. do {
  3472. add_preempt_count_notrace(PREEMPT_ACTIVE);
  3473. schedule();
  3474. sub_preempt_count_notrace(PREEMPT_ACTIVE);
  3475. /*
  3476. * Check again in case we missed a preemption opportunity
  3477. * between schedule and now.
  3478. */
  3479. barrier();
  3480. } while (need_resched());
  3481. }
  3482. EXPORT_SYMBOL(preempt_schedule);
  3483. /*
  3484. * this is the entry point to schedule() from kernel preemption
  3485. * off of irq context.
  3486. * Note, that this is called and return with irqs disabled. This will
  3487. * protect us against recursive calling from irq.
  3488. */
  3489. asmlinkage void __sched preempt_schedule_irq(void)
  3490. {
  3491. struct thread_info *ti = current_thread_info();
  3492. /* Catch callers which need to be fixed */
  3493. BUG_ON(ti->preempt_count || !irqs_disabled());
  3494. do {
  3495. add_preempt_count(PREEMPT_ACTIVE);
  3496. local_irq_enable();
  3497. schedule();
  3498. local_irq_disable();
  3499. sub_preempt_count(PREEMPT_ACTIVE);
  3500. /*
  3501. * Check again in case we missed a preemption opportunity
  3502. * between schedule and now.
  3503. */
  3504. barrier();
  3505. } while (need_resched());
  3506. }
  3507. #endif /* CONFIG_PREEMPT */
  3508. int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
  3509. void *key)
  3510. {
  3511. return try_to_wake_up(curr->private, mode, wake_flags);
  3512. }
  3513. EXPORT_SYMBOL(default_wake_function);
  3514. /*
  3515. * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
  3516. * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
  3517. * number) then we wake all the non-exclusive tasks and one exclusive task.
  3518. *
  3519. * There are circumstances in which we can try to wake a task which has already
  3520. * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
  3521. * zero in this (rare) case, and we handle it by continuing to scan the queue.
  3522. */
  3523. static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
  3524. int nr_exclusive, int wake_flags, void *key)
  3525. {
  3526. wait_queue_t *curr, *next;
  3527. list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
  3528. unsigned flags = curr->flags;
  3529. if (curr->func(curr, mode, wake_flags, key) &&
  3530. (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
  3531. break;
  3532. }
  3533. }
  3534. /**
  3535. * __wake_up - wake up threads blocked on a waitqueue.
  3536. * @q: the waitqueue
  3537. * @mode: which threads
  3538. * @nr_exclusive: how many wake-one or wake-many threads to wake up
  3539. * @key: is directly passed to the wakeup function
  3540. *
  3541. * It may be assumed that this function implies a write memory barrier before
  3542. * changing the task state if and only if any tasks are woken up.
  3543. */
  3544. void __wake_up(wait_queue_head_t *q, unsigned int mode,
  3545. int nr_exclusive, void *key)
  3546. {
  3547. unsigned long flags;
  3548. spin_lock_irqsave(&q->lock, flags);
  3549. __wake_up_common(q, mode, nr_exclusive, 0, key);
  3550. spin_unlock_irqrestore(&q->lock, flags);
  3551. }
  3552. EXPORT_SYMBOL(__wake_up);
  3553. /*
  3554. * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
  3555. */
  3556. void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
  3557. {
  3558. __wake_up_common(q, mode, 1, 0, NULL);
  3559. }
  3560. EXPORT_SYMBOL_GPL(__wake_up_locked);
  3561. void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)
  3562. {
  3563. __wake_up_common(q, mode, 1, 0, key);
  3564. }
  3565. /**
  3566. * __wake_up_sync_key - wake up threads blocked on a waitqueue.
  3567. * @q: the waitqueue
  3568. * @mode: which threads
  3569. * @nr_exclusive: how many wake-one or wake-many threads to wake up
  3570. * @key: opaque value to be passed to wakeup targets
  3571. *
  3572. * The sync wakeup differs that the waker knows that it will schedule
  3573. * away soon, so while the target thread will be woken up, it will not
  3574. * be migrated to another CPU - ie. the two threads are 'synchronized'
  3575. * with each other. This can prevent needless bouncing between CPUs.
  3576. *
  3577. * On UP it can prevent extra preemption.
  3578. *
  3579. * It may be assumed that this function implies a write memory barrier before
  3580. * changing the task state if and only if any tasks are woken up.
  3581. */
  3582. void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
  3583. int nr_exclusive, void *key)
  3584. {
  3585. unsigned long flags;
  3586. int wake_flags = WF_SYNC;
  3587. if (unlikely(!q))
  3588. return;
  3589. if (unlikely(!nr_exclusive))
  3590. wake_flags = 0;
  3591. spin_lock_irqsave(&q->lock, flags);
  3592. __wake_up_common(q, mode, nr_exclusive, wake_flags, key);
  3593. spin_unlock_irqrestore(&q->lock, flags);
  3594. }
  3595. EXPORT_SYMBOL_GPL(__wake_up_sync_key);
  3596. /*
  3597. * __wake_up_sync - see __wake_up_sync_key()
  3598. */
  3599. void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
  3600. {
  3601. __wake_up_sync_key(q, mode, nr_exclusive, NULL);
  3602. }
  3603. EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */
  3604. /**
  3605. * complete: - signals a single thread waiting on this completion
  3606. * @x: holds the state of this particular completion
  3607. *
  3608. * This will wake up a single thread waiting on this completion. Threads will be
  3609. * awakened in the same order in which they were queued.
  3610. *
  3611. * See also complete_all(), wait_for_completion() and related routines.
  3612. *
  3613. * It may be assumed that this function implies a write memory barrier before
  3614. * changing the task state if and only if any tasks are woken up.
  3615. */
  3616. void complete(struct completion *x)
  3617. {
  3618. unsigned long flags;
  3619. spin_lock_irqsave(&x->wait.lock, flags);
  3620. x->done++;
  3621. __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
  3622. spin_unlock_irqrestore(&x->wait.lock, flags);
  3623. }
  3624. EXPORT_SYMBOL(complete);
  3625. /**
  3626. * complete_all: - signals all threads waiting on this completion
  3627. * @x: holds the state of this particular completion
  3628. *
  3629. * This will wake up all threads waiting on this particular completion event.
  3630. *
  3631. * It may be assumed that this function implies a write memory barrier before
  3632. * changing the task state if and only if any tasks are woken up.
  3633. */
  3634. void complete_all(struct completion *x)
  3635. {
  3636. unsigned long flags;
  3637. spin_lock_irqsave(&x->wait.lock, flags);
  3638. x->done += UINT_MAX/2;
  3639. __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
  3640. spin_unlock_irqrestore(&x->wait.lock, flags);
  3641. }
  3642. EXPORT_SYMBOL(complete_all);
  3643. static inline long __sched
  3644. do_wait_for_common(struct completion *x, long timeout, int state)
  3645. {
  3646. if (!x->done) {
  3647. DECLARE_WAITQUEUE(wait, current);
  3648. __add_wait_queue_tail_exclusive(&x->wait, &wait);
  3649. do {
  3650. if (signal_pending_state(state, current)) {
  3651. timeout = -ERESTARTSYS;
  3652. break;
  3653. }
  3654. __set_current_state(state);
  3655. spin_unlock_irq(&x->wait.lock);
  3656. timeout = schedule_timeout(timeout);
  3657. spin_lock_irq(&x->wait.lock);
  3658. } while (!x->done && timeout);
  3659. __remove_wait_queue(&x->wait, &wait);
  3660. if (!x->done)
  3661. return timeout;
  3662. }
  3663. x->done--;
  3664. return timeout ?: 1;
  3665. }
  3666. static long __sched
  3667. wait_for_common(struct completion *x, long timeout, int state)
  3668. {
  3669. might_sleep();
  3670. spin_lock_irq(&x->wait.lock);
  3671. timeout = do_wait_for_common(x, timeout, state);
  3672. spin_unlock_irq(&x->wait.lock);
  3673. return timeout;
  3674. }
  3675. /**
  3676. * wait_for_completion: - waits for completion of a task
  3677. * @x: holds the state of this particular completion
  3678. *
  3679. * This waits to be signaled for completion of a specific task. It is NOT
  3680. * interruptible and there is no timeout.
  3681. *
  3682. * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
  3683. * and interrupt capability. Also see complete().
  3684. */
  3685. void __sched wait_for_completion(struct completion *x)
  3686. {
  3687. wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
  3688. }
  3689. EXPORT_SYMBOL(wait_for_completion);
  3690. /**
  3691. * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
  3692. * @x: holds the state of this particular completion
  3693. * @timeout: timeout value in jiffies
  3694. *
  3695. * This waits for either a completion of a specific task to be signaled or for a
  3696. * specified timeout to expire. The timeout is in jiffies. It is not
  3697. * interruptible.
  3698. */
  3699. unsigned long __sched
  3700. wait_for_completion_timeout(struct completion *x, unsigned long timeout)
  3701. {
  3702. return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
  3703. }
  3704. EXPORT_SYMBOL(wait_for_completion_timeout);
  3705. /**
  3706. * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
  3707. * @x: holds the state of this particular completion
  3708. *
  3709. * This waits for completion of a specific task to be signaled. It is
  3710. * interruptible.
  3711. */
  3712. int __sched wait_for_completion_interruptible(struct completion *x)
  3713. {
  3714. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
  3715. if (t == -ERESTARTSYS)
  3716. return t;
  3717. return 0;
  3718. }
  3719. EXPORT_SYMBOL(wait_for_completion_interruptible);
  3720. /**
  3721. * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
  3722. * @x: holds the state of this particular completion
  3723. * @timeout: timeout value in jiffies
  3724. *
  3725. * This waits for either a completion of a specific task to be signaled or for a
  3726. * specified timeout to expire. It is interruptible. The timeout is in jiffies.
  3727. */
  3728. long __sched
  3729. wait_for_completion_interruptible_timeout(struct completion *x,
  3730. unsigned long timeout)
  3731. {
  3732. return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
  3733. }
  3734. EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
  3735. /**
  3736. * wait_for_completion_killable: - waits for completion of a task (killable)
  3737. * @x: holds the state of this particular completion
  3738. *
  3739. * This waits to be signaled for completion of a specific task. It can be
  3740. * interrupted by a kill signal.
  3741. */
  3742. int __sched wait_for_completion_killable(struct completion *x)
  3743. {
  3744. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
  3745. if (t == -ERESTARTSYS)
  3746. return t;
  3747. return 0;
  3748. }
  3749. EXPORT_SYMBOL(wait_for_completion_killable);
  3750. /**
  3751. * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
  3752. * @x: holds the state of this particular completion
  3753. * @timeout: timeout value in jiffies
  3754. *
  3755. * This waits for either a completion of a specific task to be
  3756. * signaled or for a specified timeout to expire. It can be
  3757. * interrupted by a kill signal. The timeout is in jiffies.
  3758. */
  3759. long __sched
  3760. wait_for_completion_killable_timeout(struct completion *x,
  3761. unsigned long timeout)
  3762. {
  3763. return wait_for_common(x, timeout, TASK_KILLABLE);
  3764. }
  3765. EXPORT_SYMBOL(wait_for_completion_killable_timeout);
  3766. /**
  3767. * try_wait_for_completion - try to decrement a completion without blocking
  3768. * @x: completion structure
  3769. *
  3770. * Returns: 0 if a decrement cannot be done without blocking
  3771. * 1 if a decrement succeeded.
  3772. *
  3773. * If a completion is being used as a counting completion,
  3774. * attempt to decrement the counter without blocking. This
  3775. * enables us to avoid waiting if the resource the completion
  3776. * is protecting is not available.
  3777. */
  3778. bool try_wait_for_completion(struct completion *x)
  3779. {
  3780. unsigned long flags;
  3781. int ret = 1;
  3782. spin_lock_irqsave(&x->wait.lock, flags);
  3783. if (!x->done)
  3784. ret = 0;
  3785. else
  3786. x->done--;
  3787. spin_unlock_irqrestore(&x->wait.lock, flags);
  3788. return ret;
  3789. }
  3790. EXPORT_SYMBOL(try_wait_for_completion);
  3791. /**
  3792. * completion_done - Test to see if a completion has any waiters
  3793. * @x: completion structure
  3794. *
  3795. * Returns: 0 if there are waiters (wait_for_completion() in progress)
  3796. * 1 if there are no waiters.
  3797. *
  3798. */
  3799. bool completion_done(struct completion *x)
  3800. {
  3801. unsigned long flags;
  3802. int ret = 1;
  3803. spin_lock_irqsave(&x->wait.lock, flags);
  3804. if (!x->done)
  3805. ret = 0;
  3806. spin_unlock_irqrestore(&x->wait.lock, flags);
  3807. return ret;
  3808. }
  3809. EXPORT_SYMBOL(completion_done);
  3810. static long __sched
  3811. sleep_on_common(wait_queue_head_t *q, int state, long timeout)
  3812. {
  3813. unsigned long flags;
  3814. wait_queue_t wait;
  3815. init_waitqueue_entry(&wait, current);
  3816. __set_current_state(state);
  3817. spin_lock_irqsave(&q->lock, flags);
  3818. __add_wait_queue(q, &wait);
  3819. spin_unlock(&q->lock);
  3820. timeout = schedule_timeout(timeout);
  3821. spin_lock_irq(&q->lock);
  3822. __remove_wait_queue(q, &wait);
  3823. spin_unlock_irqrestore(&q->lock, flags);
  3824. return timeout;
  3825. }
  3826. void __sched interruptible_sleep_on(wait_queue_head_t *q)
  3827. {
  3828. sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
  3829. }
  3830. EXPORT_SYMBOL(interruptible_sleep_on);
  3831. long __sched
  3832. interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
  3833. {
  3834. return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
  3835. }
  3836. EXPORT_SYMBOL(interruptible_sleep_on_timeout);
  3837. void __sched sleep_on(wait_queue_head_t *q)
  3838. {
  3839. sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
  3840. }
  3841. EXPORT_SYMBOL(sleep_on);
  3842. long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
  3843. {
  3844. return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
  3845. }
  3846. EXPORT_SYMBOL(sleep_on_timeout);
  3847. #ifdef CONFIG_RT_MUTEXES
  3848. /*
  3849. * rt_mutex_setprio - set the current priority of a task
  3850. * @p: task
  3851. * @prio: prio value (kernel-internal form)
  3852. *
  3853. * This function changes the 'effective' priority of a task. It does
  3854. * not touch ->normal_prio like __setscheduler().
  3855. *
  3856. * Used by the rt_mutex code to implement priority inheritance logic.
  3857. */
  3858. void rt_mutex_setprio(struct task_struct *p, int prio)
  3859. {
  3860. unsigned long flags;
  3861. int oldprio, on_rq, running;
  3862. struct rq *rq;
  3863. const struct sched_class *prev_class;
  3864. BUG_ON(prio < 0 || prio > MAX_PRIO);
  3865. rq = task_rq_lock(p, &flags);
  3866. trace_sched_pi_setprio(p, prio);
  3867. oldprio = p->prio;
  3868. prev_class = p->sched_class;
  3869. on_rq = p->se.on_rq;
  3870. running = task_current(rq, p);
  3871. if (on_rq)
  3872. dequeue_task(rq, p, 0);
  3873. if (running)
  3874. p->sched_class->put_prev_task(rq, p);
  3875. if (rt_prio(prio))
  3876. p->sched_class = &rt_sched_class;
  3877. else
  3878. p->sched_class = &fair_sched_class;
  3879. p->prio = prio;
  3880. if (running)
  3881. p->sched_class->set_curr_task(rq);
  3882. if (on_rq) {
  3883. enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0);
  3884. check_class_changed(rq, p, prev_class, oldprio, running);
  3885. }
  3886. task_rq_unlock(rq, &flags);
  3887. }
  3888. #endif
  3889. void set_user_nice(struct task_struct *p, long nice)
  3890. {
  3891. int old_prio, delta, on_rq;
  3892. unsigned long flags;
  3893. struct rq *rq;
  3894. if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
  3895. return;
  3896. /*
  3897. * We have to be careful, if called from sys_setpriority(),
  3898. * the task might be in the middle of scheduling on another CPU.
  3899. */
  3900. rq = task_rq_lock(p, &flags);
  3901. /*
  3902. * The RT priorities are set via sched_setscheduler(), but we still
  3903. * allow the 'normal' nice value to be set - but as expected
  3904. * it wont have any effect on scheduling until the task is
  3905. * SCHED_FIFO/SCHED_RR:
  3906. */
  3907. if (task_has_rt_policy(p)) {
  3908. p->static_prio = NICE_TO_PRIO(nice);
  3909. goto out_unlock;
  3910. }
  3911. on_rq = p->se.on_rq;
  3912. if (on_rq)
  3913. dequeue_task(rq, p, 0);
  3914. p->static_prio = NICE_TO_PRIO(nice);
  3915. set_load_weight(p);
  3916. old_prio = p->prio;
  3917. p->prio = effective_prio(p);
  3918. delta = p->prio - old_prio;
  3919. if (on_rq) {
  3920. enqueue_task(rq, p, 0);
  3921. /*
  3922. * If the task increased its priority or is running and
  3923. * lowered its priority, then reschedule its CPU:
  3924. */
  3925. if (delta < 0 || (delta > 0 && task_running(rq, p)))
  3926. resched_task(rq->curr);
  3927. }
  3928. out_unlock:
  3929. task_rq_unlock(rq, &flags);
  3930. }
  3931. EXPORT_SYMBOL(set_user_nice);
  3932. /*
  3933. * can_nice - check if a task can reduce its nice value
  3934. * @p: task
  3935. * @nice: nice value
  3936. */
  3937. int can_nice(const struct task_struct *p, const int nice)
  3938. {
  3939. /* convert nice value [19,-20] to rlimit style value [1,40] */
  3940. int nice_rlim = 20 - nice;
  3941. return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
  3942. capable(CAP_SYS_NICE));
  3943. }
  3944. #ifdef __ARCH_WANT_SYS_NICE
  3945. /*
  3946. * sys_nice - change the priority of the current process.
  3947. * @increment: priority increment
  3948. *
  3949. * sys_setpriority is a more generic, but much slower function that
  3950. * does similar things.
  3951. */
  3952. SYSCALL_DEFINE1(nice, int, increment)
  3953. {
  3954. long nice, retval;
  3955. /*
  3956. * Setpriority might change our priority at the same moment.
  3957. * We don't have to worry. Conceptually one call occurs first
  3958. * and we have a single winner.
  3959. */
  3960. if (increment < -40)
  3961. increment = -40;
  3962. if (increment > 40)
  3963. increment = 40;
  3964. nice = TASK_NICE(current) + increment;
  3965. if (nice < -20)
  3966. nice = -20;
  3967. if (nice > 19)
  3968. nice = 19;
  3969. if (increment < 0 && !can_nice(current, nice))
  3970. return -EPERM;
  3971. retval = security_task_setnice(current, nice);
  3972. if (retval)
  3973. return retval;
  3974. set_user_nice(current, nice);
  3975. return 0;
  3976. }
  3977. #endif
  3978. /**
  3979. * task_prio - return the priority value of a given task.
  3980. * @p: the task in question.
  3981. *
  3982. * This is the priority value as seen by users in /proc.
  3983. * RT tasks are offset by -200. Normal tasks are centered
  3984. * around 0, value goes from -16 to +15.
  3985. */
  3986. int task_prio(const struct task_struct *p)
  3987. {
  3988. return p->prio - MAX_RT_PRIO;
  3989. }
  3990. /**
  3991. * task_nice - return the nice value of a given task.
  3992. * @p: the task in question.
  3993. */
  3994. int task_nice(const struct task_struct *p)
  3995. {
  3996. return TASK_NICE(p);
  3997. }
  3998. EXPORT_SYMBOL(task_nice);
  3999. /**
  4000. * idle_cpu - is a given cpu idle currently?
  4001. * @cpu: the processor in question.
  4002. */
  4003. int idle_cpu(int cpu)
  4004. {
  4005. return cpu_curr(cpu) == cpu_rq(cpu)->idle;
  4006. }
  4007. /**
  4008. * idle_task - return the idle task for a given cpu.
  4009. * @cpu: the processor in question.
  4010. */
  4011. struct task_struct *idle_task(int cpu)
  4012. {
  4013. return cpu_rq(cpu)->idle;
  4014. }
  4015. /**
  4016. * find_process_by_pid - find a process with a matching PID value.
  4017. * @pid: the pid in question.
  4018. */
  4019. static struct task_struct *find_process_by_pid(pid_t pid)
  4020. {
  4021. return pid ? find_task_by_vpid(pid) : current;
  4022. }
  4023. /* Actually do priority change: must hold rq lock. */
  4024. static void
  4025. __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
  4026. {
  4027. BUG_ON(p->se.on_rq);
  4028. p->policy = policy;
  4029. p->rt_priority = prio;
  4030. p->normal_prio = normal_prio(p);
  4031. /* we are holding p->pi_lock already */
  4032. p->prio = rt_mutex_getprio(p);
  4033. if (rt_prio(p->prio))
  4034. p->sched_class = &rt_sched_class;
  4035. else
  4036. p->sched_class = &fair_sched_class;
  4037. set_load_weight(p);
  4038. }
  4039. /*
  4040. * check the target process has a UID that matches the current process's
  4041. */
  4042. static bool check_same_owner(struct task_struct *p)
  4043. {
  4044. const struct cred *cred = current_cred(), *pcred;
  4045. bool match;
  4046. rcu_read_lock();
  4047. pcred = __task_cred(p);
  4048. match = (cred->euid == pcred->euid ||
  4049. cred->euid == pcred->uid);
  4050. rcu_read_unlock();
  4051. return match;
  4052. }
  4053. static int __sched_setscheduler(struct task_struct *p, int policy,
  4054. const struct sched_param *param, bool user)
  4055. {
  4056. int retval, oldprio, oldpolicy = -1, on_rq, running;
  4057. unsigned long flags;
  4058. const struct sched_class *prev_class;
  4059. struct rq *rq;
  4060. int reset_on_fork;
  4061. /* may grab non-irq protected spin_locks */
  4062. BUG_ON(in_interrupt());
  4063. recheck:
  4064. /* double check policy once rq lock held */
  4065. if (policy < 0) {
  4066. reset_on_fork = p->sched_reset_on_fork;
  4067. policy = oldpolicy = p->policy;
  4068. } else {
  4069. reset_on_fork = !!(policy & SCHED_RESET_ON_FORK);
  4070. policy &= ~SCHED_RESET_ON_FORK;
  4071. if (policy != SCHED_FIFO && policy != SCHED_RR &&
  4072. policy != SCHED_NORMAL && policy != SCHED_BATCH &&
  4073. policy != SCHED_IDLE)
  4074. return -EINVAL;
  4075. }
  4076. /*
  4077. * Valid priorities for SCHED_FIFO and SCHED_RR are
  4078. * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
  4079. * SCHED_BATCH and SCHED_IDLE is 0.
  4080. */
  4081. if (param->sched_priority < 0 ||
  4082. (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
  4083. (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
  4084. return -EINVAL;
  4085. if (rt_policy(policy) != (param->sched_priority != 0))
  4086. return -EINVAL;
  4087. /*
  4088. * Allow unprivileged RT tasks to decrease priority:
  4089. */
  4090. if (user && !capable(CAP_SYS_NICE)) {
  4091. if (rt_policy(policy)) {
  4092. unsigned long rlim_rtprio =
  4093. task_rlimit(p, RLIMIT_RTPRIO);
  4094. /* can't set/change the rt policy */
  4095. if (policy != p->policy && !rlim_rtprio)
  4096. return -EPERM;
  4097. /* can't increase priority */
  4098. if (param->sched_priority > p->rt_priority &&
  4099. param->sched_priority > rlim_rtprio)
  4100. return -EPERM;
  4101. }
  4102. /*
  4103. * Like positive nice levels, dont allow tasks to
  4104. * move out of SCHED_IDLE either:
  4105. */
  4106. if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
  4107. return -EPERM;
  4108. /* can't change other user's priorities */
  4109. if (!check_same_owner(p))
  4110. return -EPERM;
  4111. /* Normal users shall not reset the sched_reset_on_fork flag */
  4112. if (p->sched_reset_on_fork && !reset_on_fork)
  4113. return -EPERM;
  4114. }
  4115. if (user) {
  4116. retval = security_task_setscheduler(p);
  4117. if (retval)
  4118. return retval;
  4119. }
  4120. /*
  4121. * make sure no PI-waiters arrive (or leave) while we are
  4122. * changing the priority of the task:
  4123. */
  4124. raw_spin_lock_irqsave(&p->pi_lock, flags);
  4125. /*
  4126. * To be able to change p->policy safely, the apropriate
  4127. * runqueue lock must be held.
  4128. */
  4129. rq = __task_rq_lock(p);
  4130. /*
  4131. * Changing the policy of the stop threads its a very bad idea
  4132. */
  4133. if (p == rq->stop) {
  4134. __task_rq_unlock(rq);
  4135. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  4136. return -EINVAL;
  4137. }
  4138. #ifdef CONFIG_RT_GROUP_SCHED
  4139. if (user) {
  4140. /*
  4141. * Do not allow realtime tasks into groups that have no runtime
  4142. * assigned.
  4143. */
  4144. if (rt_bandwidth_enabled() && rt_policy(policy) &&
  4145. task_group(p)->rt_bandwidth.rt_runtime == 0) {
  4146. __task_rq_unlock(rq);
  4147. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  4148. return -EPERM;
  4149. }
  4150. }
  4151. #endif
  4152. /* recheck policy now with rq lock held */
  4153. if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
  4154. policy = oldpolicy = -1;
  4155. __task_rq_unlock(rq);
  4156. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  4157. goto recheck;
  4158. }
  4159. on_rq = p->se.on_rq;
  4160. running = task_current(rq, p);
  4161. if (on_rq)
  4162. deactivate_task(rq, p, 0);
  4163. if (running)
  4164. p->sched_class->put_prev_task(rq, p);
  4165. p->sched_reset_on_fork = reset_on_fork;
  4166. oldprio = p->prio;
  4167. prev_class = p->sched_class;
  4168. __setscheduler(rq, p, policy, param->sched_priority);
  4169. if (running)
  4170. p->sched_class->set_curr_task(rq);
  4171. if (on_rq) {
  4172. activate_task(rq, p, 0);
  4173. check_class_changed(rq, p, prev_class, oldprio, running);
  4174. }
  4175. __task_rq_unlock(rq);
  4176. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  4177. rt_mutex_adjust_pi(p);
  4178. return 0;
  4179. }
  4180. /**
  4181. * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
  4182. * @p: the task in question.
  4183. * @policy: new policy.
  4184. * @param: structure containing the new RT priority.
  4185. *
  4186. * NOTE that the task may be already dead.
  4187. */
  4188. int sched_setscheduler(struct task_struct *p, int policy,
  4189. const struct sched_param *param)
  4190. {
  4191. return __sched_setscheduler(p, policy, param, true);
  4192. }
  4193. EXPORT_SYMBOL_GPL(sched_setscheduler);
  4194. /**
  4195. * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
  4196. * @p: the task in question.
  4197. * @policy: new policy.
  4198. * @param: structure containing the new RT priority.
  4199. *
  4200. * Just like sched_setscheduler, only don't bother checking if the
  4201. * current context has permission. For example, this is needed in
  4202. * stop_machine(): we create temporary high priority worker threads,
  4203. * but our caller might not have that capability.
  4204. */
  4205. int sched_setscheduler_nocheck(struct task_struct *p, int policy,
  4206. const struct sched_param *param)
  4207. {
  4208. return __sched_setscheduler(p, policy, param, false);
  4209. }
  4210. static int
  4211. do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
  4212. {
  4213. struct sched_param lparam;
  4214. struct task_struct *p;
  4215. int retval;
  4216. if (!param || pid < 0)
  4217. return -EINVAL;
  4218. if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
  4219. return -EFAULT;
  4220. rcu_read_lock();
  4221. retval = -ESRCH;
  4222. p = find_process_by_pid(pid);
  4223. if (p != NULL)
  4224. retval = sched_setscheduler(p, policy, &lparam);
  4225. rcu_read_unlock();
  4226. return retval;
  4227. }
  4228. /**
  4229. * sys_sched_setscheduler - set/change the scheduler policy and RT priority
  4230. * @pid: the pid in question.
  4231. * @policy: new policy.
  4232. * @param: structure containing the new RT priority.
  4233. */
  4234. SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
  4235. struct sched_param __user *, param)
  4236. {
  4237. /* negative values for policy are not valid */
  4238. if (policy < 0)
  4239. return -EINVAL;
  4240. return do_sched_setscheduler(pid, policy, param);
  4241. }
  4242. /**
  4243. * sys_sched_setparam - set/change the RT priority of a thread
  4244. * @pid: the pid in question.
  4245. * @param: structure containing the new RT priority.
  4246. */
  4247. SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
  4248. {
  4249. return do_sched_setscheduler(pid, -1, param);
  4250. }
  4251. /**
  4252. * sys_sched_getscheduler - get the policy (scheduling class) of a thread
  4253. * @pid: the pid in question.
  4254. */
  4255. SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
  4256. {
  4257. struct task_struct *p;
  4258. int retval;
  4259. if (pid < 0)
  4260. return -EINVAL;
  4261. retval = -ESRCH;
  4262. rcu_read_lock();
  4263. p = find_process_by_pid(pid);
  4264. if (p) {
  4265. retval = security_task_getscheduler(p);
  4266. if (!retval)
  4267. retval = p->policy
  4268. | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
  4269. }
  4270. rcu_read_unlock();
  4271. return retval;
  4272. }
  4273. /**
  4274. * sys_sched_getparam - get the RT priority of a thread
  4275. * @pid: the pid in question.
  4276. * @param: structure containing the RT priority.
  4277. */
  4278. SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
  4279. {
  4280. struct sched_param lp;
  4281. struct task_struct *p;
  4282. int retval;
  4283. if (!param || pid < 0)
  4284. return -EINVAL;
  4285. rcu_read_lock();
  4286. p = find_process_by_pid(pid);
  4287. retval = -ESRCH;
  4288. if (!p)
  4289. goto out_unlock;
  4290. retval = security_task_getscheduler(p);
  4291. if (retval)
  4292. goto out_unlock;
  4293. lp.sched_priority = p->rt_priority;
  4294. rcu_read_unlock();
  4295. /*
  4296. * This one might sleep, we cannot do it with a spinlock held ...
  4297. */
  4298. retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
  4299. return retval;
  4300. out_unlock:
  4301. rcu_read_unlock();
  4302. return retval;
  4303. }
  4304. long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
  4305. {
  4306. cpumask_var_t cpus_allowed, new_mask;
  4307. struct task_struct *p;
  4308. int retval;
  4309. get_online_cpus();
  4310. rcu_read_lock();
  4311. p = find_process_by_pid(pid);
  4312. if (!p) {
  4313. rcu_read_unlock();
  4314. put_online_cpus();
  4315. return -ESRCH;
  4316. }
  4317. /* Prevent p going away */
  4318. get_task_struct(p);
  4319. rcu_read_unlock();
  4320. if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
  4321. retval = -ENOMEM;
  4322. goto out_put_task;
  4323. }
  4324. if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
  4325. retval = -ENOMEM;
  4326. goto out_free_cpus_allowed;
  4327. }
  4328. retval = -EPERM;
  4329. if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
  4330. goto out_unlock;
  4331. retval = security_task_setscheduler(p);
  4332. if (retval)
  4333. goto out_unlock;
  4334. cpuset_cpus_allowed(p, cpus_allowed);
  4335. cpumask_and(new_mask, in_mask, cpus_allowed);
  4336. again:
  4337. retval = set_cpus_allowed_ptr(p, new_mask);
  4338. if (!retval) {
  4339. cpuset_cpus_allowed(p, cpus_allowed);
  4340. if (!cpumask_subset(new_mask, cpus_allowed)) {
  4341. /*
  4342. * We must have raced with a concurrent cpuset
  4343. * update. Just reset the cpus_allowed to the
  4344. * cpuset's cpus_allowed
  4345. */
  4346. cpumask_copy(new_mask, cpus_allowed);
  4347. goto again;
  4348. }
  4349. }
  4350. out_unlock:
  4351. free_cpumask_var(new_mask);
  4352. out_free_cpus_allowed:
  4353. free_cpumask_var(cpus_allowed);
  4354. out_put_task:
  4355. put_task_struct(p);
  4356. put_online_cpus();
  4357. return retval;
  4358. }
  4359. static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
  4360. struct cpumask *new_mask)
  4361. {
  4362. if (len < cpumask_size())
  4363. cpumask_clear(new_mask);
  4364. else if (len > cpumask_size())
  4365. len = cpumask_size();
  4366. return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
  4367. }
  4368. /**
  4369. * sys_sched_setaffinity - set the cpu affinity of a process
  4370. * @pid: pid of the process
  4371. * @len: length in bytes of the bitmask pointed to by user_mask_ptr
  4372. * @user_mask_ptr: user-space pointer to the new cpu mask
  4373. */
  4374. SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
  4375. unsigned long __user *, user_mask_ptr)
  4376. {
  4377. cpumask_var_t new_mask;
  4378. int retval;
  4379. if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
  4380. return -ENOMEM;
  4381. retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
  4382. if (retval == 0)
  4383. retval = sched_setaffinity(pid, new_mask);
  4384. free_cpumask_var(new_mask);
  4385. return retval;
  4386. }
  4387. long sched_getaffinity(pid_t pid, struct cpumask *mask)
  4388. {
  4389. struct task_struct *p;
  4390. unsigned long flags;
  4391. struct rq *rq;
  4392. int retval;
  4393. get_online_cpus();
  4394. rcu_read_lock();
  4395. retval = -ESRCH;
  4396. p = find_process_by_pid(pid);
  4397. if (!p)
  4398. goto out_unlock;
  4399. retval = security_task_getscheduler(p);
  4400. if (retval)
  4401. goto out_unlock;
  4402. rq = task_rq_lock(p, &flags);
  4403. cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
  4404. task_rq_unlock(rq, &flags);
  4405. out_unlock:
  4406. rcu_read_unlock();
  4407. put_online_cpus();
  4408. return retval;
  4409. }
  4410. /**
  4411. * sys_sched_getaffinity - get the cpu affinity of a process
  4412. * @pid: pid of the process
  4413. * @len: length in bytes of the bitmask pointed to by user_mask_ptr
  4414. * @user_mask_ptr: user-space pointer to hold the current cpu mask
  4415. */
  4416. SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
  4417. unsigned long __user *, user_mask_ptr)
  4418. {
  4419. int ret;
  4420. cpumask_var_t mask;
  4421. if ((len * BITS_PER_BYTE) < nr_cpu_ids)
  4422. return -EINVAL;
  4423. if (len & (sizeof(unsigned long)-1))
  4424. return -EINVAL;
  4425. if (!alloc_cpumask_var(&mask, GFP_KERNEL))
  4426. return -ENOMEM;
  4427. ret = sched_getaffinity(pid, mask);
  4428. if (ret == 0) {
  4429. size_t retlen = min_t(size_t, len, cpumask_size());
  4430. if (copy_to_user(user_mask_ptr, mask, retlen))
  4431. ret = -EFAULT;
  4432. else
  4433. ret = retlen;
  4434. }
  4435. free_cpumask_var(mask);
  4436. return ret;
  4437. }
  4438. /**
  4439. * sys_sched_yield - yield the current processor to other threads.
  4440. *
  4441. * This function yields the current CPU to other tasks. If there are no
  4442. * other threads running on this CPU then this function will return.
  4443. */
  4444. SYSCALL_DEFINE0(sched_yield)
  4445. {
  4446. struct rq *rq = this_rq_lock();
  4447. schedstat_inc(rq, yld_count);
  4448. current->sched_class->yield_task(rq);
  4449. /*
  4450. * Since we are going to call schedule() anyway, there's
  4451. * no need to preempt or enable interrupts:
  4452. */
  4453. __release(rq->lock);
  4454. spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
  4455. do_raw_spin_unlock(&rq->lock);
  4456. preempt_enable_no_resched();
  4457. schedule();
  4458. return 0;
  4459. }
  4460. static inline int should_resched(void)
  4461. {
  4462. return need_resched() && !(preempt_count() & PREEMPT_ACTIVE);
  4463. }
  4464. static void __cond_resched(void)
  4465. {
  4466. add_preempt_count(PREEMPT_ACTIVE);
  4467. schedule();
  4468. sub_preempt_count(PREEMPT_ACTIVE);
  4469. }
  4470. int __sched _cond_resched(void)
  4471. {
  4472. if (should_resched()) {
  4473. __cond_resched();
  4474. return 1;
  4475. }
  4476. return 0;
  4477. }
  4478. EXPORT_SYMBOL(_cond_resched);
  4479. /*
  4480. * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
  4481. * call schedule, and on return reacquire the lock.
  4482. *
  4483. * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
  4484. * operations here to prevent schedule() from being called twice (once via
  4485. * spin_unlock(), once by hand).
  4486. */
  4487. int __cond_resched_lock(spinlock_t *lock)
  4488. {
  4489. int resched = should_resched();
  4490. int ret = 0;
  4491. lockdep_assert_held(lock);
  4492. if (spin_needbreak(lock) || resched) {
  4493. spin_unlock(lock);
  4494. if (resched)
  4495. __cond_resched();
  4496. else
  4497. cpu_relax();
  4498. ret = 1;
  4499. spin_lock(lock);
  4500. }
  4501. return ret;
  4502. }
  4503. EXPORT_SYMBOL(__cond_resched_lock);
  4504. int __sched __cond_resched_softirq(void)
  4505. {
  4506. BUG_ON(!in_softirq());
  4507. if (should_resched()) {
  4508. local_bh_enable();
  4509. __cond_resched();
  4510. local_bh_disable();
  4511. return 1;
  4512. }
  4513. return 0;
  4514. }
  4515. EXPORT_SYMBOL(__cond_resched_softirq);
  4516. /**
  4517. * yield - yield the current processor to other threads.
  4518. *
  4519. * This is a shortcut for kernel-space yielding - it marks the
  4520. * thread runnable and calls sys_sched_yield().
  4521. */
  4522. void __sched yield(void)
  4523. {
  4524. set_current_state(TASK_RUNNING);
  4525. sys_sched_yield();
  4526. }
  4527. EXPORT_SYMBOL(yield);
  4528. /*
  4529. * This task is about to go to sleep on IO. Increment rq->nr_iowait so
  4530. * that process accounting knows that this is a task in IO wait state.
  4531. */
  4532. void __sched io_schedule(void)
  4533. {
  4534. struct rq *rq = raw_rq();
  4535. delayacct_blkio_start();
  4536. atomic_inc(&rq->nr_iowait);
  4537. current->in_iowait = 1;
  4538. schedule();
  4539. current->in_iowait = 0;
  4540. atomic_dec(&rq->nr_iowait);
  4541. delayacct_blkio_end();
  4542. }
  4543. EXPORT_SYMBOL(io_schedule);
  4544. long __sched io_schedule_timeout(long timeout)
  4545. {
  4546. struct rq *rq = raw_rq();
  4547. long ret;
  4548. delayacct_blkio_start();
  4549. atomic_inc(&rq->nr_iowait);
  4550. current->in_iowait = 1;
  4551. ret = schedule_timeout(timeout);
  4552. current->in_iowait = 0;
  4553. atomic_dec(&rq->nr_iowait);
  4554. delayacct_blkio_end();
  4555. return ret;
  4556. }
  4557. /**
  4558. * sys_sched_get_priority_max - return maximum RT priority.
  4559. * @policy: scheduling class.
  4560. *
  4561. * this syscall returns the maximum rt_priority that can be used
  4562. * by a given scheduling class.
  4563. */
  4564. SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
  4565. {
  4566. int ret = -EINVAL;
  4567. switch (policy) {
  4568. case SCHED_FIFO:
  4569. case SCHED_RR:
  4570. ret = MAX_USER_RT_PRIO-1;
  4571. break;
  4572. case SCHED_NORMAL:
  4573. case SCHED_BATCH:
  4574. case SCHED_IDLE:
  4575. ret = 0;
  4576. break;
  4577. }
  4578. return ret;
  4579. }
  4580. /**
  4581. * sys_sched_get_priority_min - return minimum RT priority.
  4582. * @policy: scheduling class.
  4583. *
  4584. * this syscall returns the minimum rt_priority that can be used
  4585. * by a given scheduling class.
  4586. */
  4587. SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
  4588. {
  4589. int ret = -EINVAL;
  4590. switch (policy) {
  4591. case SCHED_FIFO:
  4592. case SCHED_RR:
  4593. ret = 1;
  4594. break;
  4595. case SCHED_NORMAL:
  4596. case SCHED_BATCH:
  4597. case SCHED_IDLE:
  4598. ret = 0;
  4599. }
  4600. return ret;
  4601. }
  4602. /**
  4603. * sys_sched_rr_get_interval - return the default timeslice of a process.
  4604. * @pid: pid of the process.
  4605. * @interval: userspace pointer to the timeslice value.
  4606. *
  4607. * this syscall writes the default timeslice value of a given process
  4608. * into the user-space timespec buffer. A value of '0' means infinity.
  4609. */
  4610. SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
  4611. struct timespec __user *, interval)
  4612. {
  4613. struct task_struct *p;
  4614. unsigned int time_slice;
  4615. unsigned long flags;
  4616. struct rq *rq;
  4617. int retval;
  4618. struct timespec t;
  4619. if (pid < 0)
  4620. return -EINVAL;
  4621. retval = -ESRCH;
  4622. rcu_read_lock();
  4623. p = find_process_by_pid(pid);
  4624. if (!p)
  4625. goto out_unlock;
  4626. retval = security_task_getscheduler(p);
  4627. if (retval)
  4628. goto out_unlock;
  4629. rq = task_rq_lock(p, &flags);
  4630. time_slice = p->sched_class->get_rr_interval(rq, p);
  4631. task_rq_unlock(rq, &flags);
  4632. rcu_read_unlock();
  4633. jiffies_to_timespec(time_slice, &t);
  4634. retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
  4635. return retval;
  4636. out_unlock:
  4637. rcu_read_unlock();
  4638. return retval;
  4639. }
  4640. static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
  4641. void sched_show_task(struct task_struct *p)
  4642. {
  4643. unsigned long free = 0;
  4644. unsigned state;
  4645. state = p->state ? __ffs(p->state) + 1 : 0;
  4646. printk(KERN_INFO "%-15.15s %c", p->comm,
  4647. state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
  4648. #if BITS_PER_LONG == 32
  4649. if (state == TASK_RUNNING)
  4650. printk(KERN_CONT " running ");
  4651. else
  4652. printk(KERN_CONT " %08lx ", thread_saved_pc(p));
  4653. #else
  4654. if (state == TASK_RUNNING)
  4655. printk(KERN_CONT " running task ");
  4656. else
  4657. printk(KERN_CONT " %016lx ", thread_saved_pc(p));
  4658. #endif
  4659. #ifdef CONFIG_DEBUG_STACK_USAGE
  4660. free = stack_not_used(p);
  4661. #endif
  4662. printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
  4663. task_pid_nr(p), task_pid_nr(p->real_parent),
  4664. (unsigned long)task_thread_info(p)->flags);
  4665. show_stack(p, NULL);
  4666. }
  4667. void show_state_filter(unsigned long state_filter)
  4668. {
  4669. struct task_struct *g, *p;
  4670. #if BITS_PER_LONG == 32
  4671. printk(KERN_INFO
  4672. " task PC stack pid father\n");
  4673. #else
  4674. printk(KERN_INFO
  4675. " task PC stack pid father\n");
  4676. #endif
  4677. read_lock(&tasklist_lock);
  4678. do_each_thread(g, p) {
  4679. /*
  4680. * reset the NMI-timeout, listing all files on a slow
  4681. * console might take alot of time:
  4682. */
  4683. touch_nmi_watchdog();
  4684. if (!state_filter || (p->state & state_filter))
  4685. sched_show_task(p);
  4686. } while_each_thread(g, p);
  4687. touch_all_softlockup_watchdogs();
  4688. #ifdef CONFIG_SCHED_DEBUG
  4689. sysrq_sched_debug_show();
  4690. #endif
  4691. read_unlock(&tasklist_lock);
  4692. /*
  4693. * Only show locks if all tasks are dumped:
  4694. */
  4695. if (!state_filter)
  4696. debug_show_all_locks();
  4697. }
  4698. void __cpuinit init_idle_bootup_task(struct task_struct *idle)
  4699. {
  4700. idle->sched_class = &idle_sched_class;
  4701. }
  4702. /**
  4703. * init_idle - set up an idle thread for a given CPU
  4704. * @idle: task in question
  4705. * @cpu: cpu the idle task belongs to
  4706. *
  4707. * NOTE: this function does not set the idle thread's NEED_RESCHED
  4708. * flag, to make booting more robust.
  4709. */
  4710. void __cpuinit init_idle(struct task_struct *idle, int cpu)
  4711. {
  4712. struct rq *rq = cpu_rq(cpu);
  4713. unsigned long flags;
  4714. raw_spin_lock_irqsave(&rq->lock, flags);
  4715. __sched_fork(idle);
  4716. idle->state = TASK_RUNNING;
  4717. idle->se.exec_start = sched_clock();
  4718. cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
  4719. /*
  4720. * We're having a chicken and egg problem, even though we are
  4721. * holding rq->lock, the cpu isn't yet set to this cpu so the
  4722. * lockdep check in task_group() will fail.
  4723. *
  4724. * Similar case to sched_fork(). / Alternatively we could
  4725. * use task_rq_lock() here and obtain the other rq->lock.
  4726. *
  4727. * Silence PROVE_RCU
  4728. */
  4729. rcu_read_lock();
  4730. __set_task_cpu(idle, cpu);
  4731. rcu_read_unlock();
  4732. rq->curr = rq->idle = idle;
  4733. #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
  4734. idle->oncpu = 1;
  4735. #endif
  4736. raw_spin_unlock_irqrestore(&rq->lock, flags);
  4737. /* Set the preempt count _outside_ the spinlocks! */
  4738. #if defined(CONFIG_PREEMPT)
  4739. task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
  4740. #else
  4741. task_thread_info(idle)->preempt_count = 0;
  4742. #endif
  4743. /*
  4744. * The idle tasks have their own, simple scheduling class:
  4745. */
  4746. idle->sched_class = &idle_sched_class;
  4747. ftrace_graph_init_task(idle);
  4748. }
  4749. /*
  4750. * In a system that switches off the HZ timer nohz_cpu_mask
  4751. * indicates which cpus entered this state. This is used
  4752. * in the rcu update to wait only for active cpus. For system
  4753. * which do not switch off the HZ timer nohz_cpu_mask should
  4754. * always be CPU_BITS_NONE.
  4755. */
  4756. cpumask_var_t nohz_cpu_mask;
  4757. /*
  4758. * Increase the granularity value when there are more CPUs,
  4759. * because with more CPUs the 'effective latency' as visible
  4760. * to users decreases. But the relationship is not linear,
  4761. * so pick a second-best guess by going with the log2 of the
  4762. * number of CPUs.
  4763. *
  4764. * This idea comes from the SD scheduler of Con Kolivas:
  4765. */
  4766. static int get_update_sysctl_factor(void)
  4767. {
  4768. unsigned int cpus = min_t(int, num_online_cpus(), 8);
  4769. unsigned int factor;
  4770. switch (sysctl_sched_tunable_scaling) {
  4771. case SCHED_TUNABLESCALING_NONE:
  4772. factor = 1;
  4773. break;
  4774. case SCHED_TUNABLESCALING_LINEAR:
  4775. factor = cpus;
  4776. break;
  4777. case SCHED_TUNABLESCALING_LOG:
  4778. default:
  4779. factor = 1 + ilog2(cpus);
  4780. break;
  4781. }
  4782. return factor;
  4783. }
  4784. static void update_sysctl(void)
  4785. {
  4786. unsigned int factor = get_update_sysctl_factor();
  4787. #define SET_SYSCTL(name) \
  4788. (sysctl_##name = (factor) * normalized_sysctl_##name)
  4789. SET_SYSCTL(sched_min_granularity);
  4790. SET_SYSCTL(sched_latency);
  4791. SET_SYSCTL(sched_wakeup_granularity);
  4792. #undef SET_SYSCTL
  4793. }
  4794. static inline void sched_init_granularity(void)
  4795. {
  4796. update_sysctl();
  4797. }
  4798. #ifdef CONFIG_SMP
  4799. /*
  4800. * This is how migration works:
  4801. *
  4802. * 1) we invoke migration_cpu_stop() on the target CPU using
  4803. * stop_one_cpu().
  4804. * 2) stopper starts to run (implicitly forcing the migrated thread
  4805. * off the CPU)
  4806. * 3) it checks whether the migrated task is still in the wrong runqueue.
  4807. * 4) if it's in the wrong runqueue then the migration thread removes
  4808. * it and puts it into the right queue.
  4809. * 5) stopper completes and stop_one_cpu() returns and the migration
  4810. * is done.
  4811. */
  4812. /*
  4813. * Change a given task's CPU affinity. Migrate the thread to a
  4814. * proper CPU and schedule it away if the CPU it's executing on
  4815. * is removed from the allowed bitmask.
  4816. *
  4817. * NOTE: the caller must have a valid reference to the task, the
  4818. * task must not exit() & deallocate itself prematurely. The
  4819. * call is not atomic; no spinlocks may be held.
  4820. */
  4821. int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
  4822. {
  4823. unsigned long flags;
  4824. struct rq *rq;
  4825. unsigned int dest_cpu;
  4826. int ret = 0;
  4827. /*
  4828. * Serialize against TASK_WAKING so that ttwu() and wunt() can
  4829. * drop the rq->lock and still rely on ->cpus_allowed.
  4830. */
  4831. again:
  4832. while (task_is_waking(p))
  4833. cpu_relax();
  4834. rq = task_rq_lock(p, &flags);
  4835. if (task_is_waking(p)) {
  4836. task_rq_unlock(rq, &flags);
  4837. goto again;
  4838. }
  4839. if (!cpumask_intersects(new_mask, cpu_active_mask)) {
  4840. ret = -EINVAL;
  4841. goto out;
  4842. }
  4843. if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
  4844. !cpumask_equal(&p->cpus_allowed, new_mask))) {
  4845. ret = -EINVAL;
  4846. goto out;
  4847. }
  4848. if (p->sched_class->set_cpus_allowed)
  4849. p->sched_class->set_cpus_allowed(p, new_mask);
  4850. else {
  4851. cpumask_copy(&p->cpus_allowed, new_mask);
  4852. p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
  4853. }
  4854. /* Can the task run on the task's current CPU? If so, we're done */
  4855. if (cpumask_test_cpu(task_cpu(p), new_mask))
  4856. goto out;
  4857. dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
  4858. if (migrate_task(p, rq)) {
  4859. struct migration_arg arg = { p, dest_cpu };
  4860. /* Need help from migration thread: drop lock and wait. */
  4861. task_rq_unlock(rq, &flags);
  4862. stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
  4863. tlb_migrate_finish(p->mm);
  4864. return 0;
  4865. }
  4866. out:
  4867. task_rq_unlock(rq, &flags);
  4868. return ret;
  4869. }
  4870. EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
  4871. /*
  4872. * Move (not current) task off this cpu, onto dest cpu. We're doing
  4873. * this because either it can't run here any more (set_cpus_allowed()
  4874. * away from this CPU, or CPU going down), or because we're
  4875. * attempting to rebalance this task on exec (sched_exec).
  4876. *
  4877. * So we race with normal scheduler movements, but that's OK, as long
  4878. * as the task is no longer on this CPU.
  4879. *
  4880. * Returns non-zero if task was successfully migrated.
  4881. */
  4882. static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
  4883. {
  4884. struct rq *rq_dest, *rq_src;
  4885. int ret = 0;
  4886. if (unlikely(!cpu_active(dest_cpu)))
  4887. return ret;
  4888. rq_src = cpu_rq(src_cpu);
  4889. rq_dest = cpu_rq(dest_cpu);
  4890. double_rq_lock(rq_src, rq_dest);
  4891. /* Already moved. */
  4892. if (task_cpu(p) != src_cpu)
  4893. goto done;
  4894. /* Affinity changed (again). */
  4895. if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
  4896. goto fail;
  4897. /*
  4898. * If we're not on a rq, the next wake-up will ensure we're
  4899. * placed properly.
  4900. */
  4901. if (p->se.on_rq) {
  4902. deactivate_task(rq_src, p, 0);
  4903. set_task_cpu(p, dest_cpu);
  4904. activate_task(rq_dest, p, 0);
  4905. check_preempt_curr(rq_dest, p, 0);
  4906. }
  4907. done:
  4908. ret = 1;
  4909. fail:
  4910. double_rq_unlock(rq_src, rq_dest);
  4911. return ret;
  4912. }
  4913. /*
  4914. * migration_cpu_stop - this will be executed by a highprio stopper thread
  4915. * and performs thread migration by bumping thread off CPU then
  4916. * 'pushing' onto another runqueue.
  4917. */
  4918. static int migration_cpu_stop(void *data)
  4919. {
  4920. struct migration_arg *arg = data;
  4921. /*
  4922. * The original target cpu might have gone down and we might
  4923. * be on another cpu but it doesn't matter.
  4924. */
  4925. local_irq_disable();
  4926. __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
  4927. local_irq_enable();
  4928. return 0;
  4929. }
  4930. #ifdef CONFIG_HOTPLUG_CPU
  4931. /*
  4932. * Ensures that the idle task is using init_mm right before its cpu goes
  4933. * offline.
  4934. */
  4935. void idle_task_exit(void)
  4936. {
  4937. struct mm_struct *mm = current->active_mm;
  4938. BUG_ON(cpu_online(smp_processor_id()));
  4939. if (mm != &init_mm)
  4940. switch_mm(mm, &init_mm, current);
  4941. mmdrop(mm);
  4942. }
  4943. /*
  4944. * While a dead CPU has no uninterruptible tasks queued at this point,
  4945. * it might still have a nonzero ->nr_uninterruptible counter, because
  4946. * for performance reasons the counter is not stricly tracking tasks to
  4947. * their home CPUs. So we just add the counter to another CPU's counter,
  4948. * to keep the global sum constant after CPU-down:
  4949. */
  4950. static void migrate_nr_uninterruptible(struct rq *rq_src)
  4951. {
  4952. struct rq *rq_dest = cpu_rq(cpumask_any(cpu_active_mask));
  4953. rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
  4954. rq_src->nr_uninterruptible = 0;
  4955. }
  4956. /*
  4957. * remove the tasks which were accounted by rq from calc_load_tasks.
  4958. */
  4959. static void calc_global_load_remove(struct rq *rq)
  4960. {
  4961. atomic_long_sub(rq->calc_load_active, &calc_load_tasks);
  4962. rq->calc_load_active = 0;
  4963. }
  4964. /*
  4965. * Migrate all tasks from the rq, sleeping tasks will be migrated by
  4966. * try_to_wake_up()->select_task_rq().
  4967. *
  4968. * Called with rq->lock held even though we'er in stop_machine() and
  4969. * there's no concurrency possible, we hold the required locks anyway
  4970. * because of lock validation efforts.
  4971. */
  4972. static void migrate_tasks(unsigned int dead_cpu)
  4973. {
  4974. struct rq *rq = cpu_rq(dead_cpu);
  4975. struct task_struct *next, *stop = rq->stop;
  4976. int dest_cpu;
  4977. /*
  4978. * Fudge the rq selection such that the below task selection loop
  4979. * doesn't get stuck on the currently eligible stop task.
  4980. *
  4981. * We're currently inside stop_machine() and the rq is either stuck
  4982. * in the stop_machine_cpu_stop() loop, or we're executing this code,
  4983. * either way we should never end up calling schedule() until we're
  4984. * done here.
  4985. */
  4986. rq->stop = NULL;
  4987. for ( ; ; ) {
  4988. /*
  4989. * There's this thread running, bail when that's the only
  4990. * remaining thread.
  4991. */
  4992. if (rq->nr_running == 1)
  4993. break;
  4994. next = pick_next_task(rq);
  4995. BUG_ON(!next);
  4996. next->sched_class->put_prev_task(rq, next);
  4997. /* Find suitable destination for @next, with force if needed. */
  4998. dest_cpu = select_fallback_rq(dead_cpu, next);
  4999. raw_spin_unlock(&rq->lock);
  5000. __migrate_task(next, dead_cpu, dest_cpu);
  5001. raw_spin_lock(&rq->lock);
  5002. }
  5003. rq->stop = stop;
  5004. }
  5005. #endif /* CONFIG_HOTPLUG_CPU */
  5006. #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
  5007. static struct ctl_table sd_ctl_dir[] = {
  5008. {
  5009. .procname = "sched_domain",
  5010. .mode = 0555,
  5011. },
  5012. {}
  5013. };
  5014. static struct ctl_table sd_ctl_root[] = {
  5015. {
  5016. .procname = "kernel",
  5017. .mode = 0555,
  5018. .child = sd_ctl_dir,
  5019. },
  5020. {}
  5021. };
  5022. static struct ctl_table *sd_alloc_ctl_entry(int n)
  5023. {
  5024. struct ctl_table *entry =
  5025. kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
  5026. return entry;
  5027. }
  5028. static void sd_free_ctl_entry(struct ctl_table **tablep)
  5029. {
  5030. struct ctl_table *entry;
  5031. /*
  5032. * In the intermediate directories, both the child directory and
  5033. * procname are dynamically allocated and could fail but the mode
  5034. * will always be set. In the lowest directory the names are
  5035. * static strings and all have proc handlers.
  5036. */
  5037. for (entry = *tablep; entry->mode; entry++) {
  5038. if (entry->child)
  5039. sd_free_ctl_entry(&entry->child);
  5040. if (entry->proc_handler == NULL)
  5041. kfree(entry->procname);
  5042. }
  5043. kfree(*tablep);
  5044. *tablep = NULL;
  5045. }
  5046. static void
  5047. set_table_entry(struct ctl_table *entry,
  5048. const char *procname, void *data, int maxlen,
  5049. mode_t mode, proc_handler *proc_handler)
  5050. {
  5051. entry->procname = procname;
  5052. entry->data = data;
  5053. entry->maxlen = maxlen;
  5054. entry->mode = mode;
  5055. entry->proc_handler = proc_handler;
  5056. }
  5057. static struct ctl_table *
  5058. sd_alloc_ctl_domain_table(struct sched_domain *sd)
  5059. {
  5060. struct ctl_table *table = sd_alloc_ctl_entry(13);
  5061. if (table == NULL)
  5062. return NULL;
  5063. set_table_entry(&table[0], "min_interval", &sd->min_interval,
  5064. sizeof(long), 0644, proc_doulongvec_minmax);
  5065. set_table_entry(&table[1], "max_interval", &sd->max_interval,
  5066. sizeof(long), 0644, proc_doulongvec_minmax);
  5067. set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
  5068. sizeof(int), 0644, proc_dointvec_minmax);
  5069. set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
  5070. sizeof(int), 0644, proc_dointvec_minmax);
  5071. set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
  5072. sizeof(int), 0644, proc_dointvec_minmax);
  5073. set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
  5074. sizeof(int), 0644, proc_dointvec_minmax);
  5075. set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
  5076. sizeof(int), 0644, proc_dointvec_minmax);
  5077. set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
  5078. sizeof(int), 0644, proc_dointvec_minmax);
  5079. set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
  5080. sizeof(int), 0644, proc_dointvec_minmax);
  5081. set_table_entry(&table[9], "cache_nice_tries",
  5082. &sd->cache_nice_tries,
  5083. sizeof(int), 0644, proc_dointvec_minmax);
  5084. set_table_entry(&table[10], "flags", &sd->flags,
  5085. sizeof(int), 0644, proc_dointvec_minmax);
  5086. set_table_entry(&table[11], "name", sd->name,
  5087. CORENAME_MAX_SIZE, 0444, proc_dostring);
  5088. /* &table[12] is terminator */
  5089. return table;
  5090. }
  5091. static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
  5092. {
  5093. struct ctl_table *entry, *table;
  5094. struct sched_domain *sd;
  5095. int domain_num = 0, i;
  5096. char buf[32];
  5097. for_each_domain(cpu, sd)
  5098. domain_num++;
  5099. entry = table = sd_alloc_ctl_entry(domain_num + 1);
  5100. if (table == NULL)
  5101. return NULL;
  5102. i = 0;
  5103. for_each_domain(cpu, sd) {
  5104. snprintf(buf, 32, "domain%d", i);
  5105. entry->procname = kstrdup(buf, GFP_KERNEL);
  5106. entry->mode = 0555;
  5107. entry->child = sd_alloc_ctl_domain_table(sd);
  5108. entry++;
  5109. i++;
  5110. }
  5111. return table;
  5112. }
  5113. static struct ctl_table_header *sd_sysctl_header;
  5114. static void register_sched_domain_sysctl(void)
  5115. {
  5116. int i, cpu_num = num_possible_cpus();
  5117. struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
  5118. char buf[32];
  5119. WARN_ON(sd_ctl_dir[0].child);
  5120. sd_ctl_dir[0].child = entry;
  5121. if (entry == NULL)
  5122. return;
  5123. for_each_possible_cpu(i) {
  5124. snprintf(buf, 32, "cpu%d", i);
  5125. entry->procname = kstrdup(buf, GFP_KERNEL);
  5126. entry->mode = 0555;
  5127. entry->child = sd_alloc_ctl_cpu_table(i);
  5128. entry++;
  5129. }
  5130. WARN_ON(sd_sysctl_header);
  5131. sd_sysctl_header = register_sysctl_table(sd_ctl_root);
  5132. }
  5133. /* may be called multiple times per register */
  5134. static void unregister_sched_domain_sysctl(void)
  5135. {
  5136. if (sd_sysctl_header)
  5137. unregister_sysctl_table(sd_sysctl_header);
  5138. sd_sysctl_header = NULL;
  5139. if (sd_ctl_dir[0].child)
  5140. sd_free_ctl_entry(&sd_ctl_dir[0].child);
  5141. }
  5142. #else
  5143. static void register_sched_domain_sysctl(void)
  5144. {
  5145. }
  5146. static void unregister_sched_domain_sysctl(void)
  5147. {
  5148. }
  5149. #endif
  5150. static void set_rq_online(struct rq *rq)
  5151. {
  5152. if (!rq->online) {
  5153. const struct sched_class *class;
  5154. cpumask_set_cpu(rq->cpu, rq->rd->online);
  5155. rq->online = 1;
  5156. for_each_class(class) {
  5157. if (class->rq_online)
  5158. class->rq_online(rq);
  5159. }
  5160. }
  5161. }
  5162. static void set_rq_offline(struct rq *rq)
  5163. {
  5164. if (rq->online) {
  5165. const struct sched_class *class;
  5166. for_each_class(class) {
  5167. if (class->rq_offline)
  5168. class->rq_offline(rq);
  5169. }
  5170. cpumask_clear_cpu(rq->cpu, rq->rd->online);
  5171. rq->online = 0;
  5172. }
  5173. }
  5174. /*
  5175. * migration_call - callback that gets triggered when a CPU is added.
  5176. * Here we can start up the necessary migration thread for the new CPU.
  5177. */
  5178. static int __cpuinit
  5179. migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
  5180. {
  5181. int cpu = (long)hcpu;
  5182. unsigned long flags;
  5183. struct rq *rq = cpu_rq(cpu);
  5184. switch (action & ~CPU_TASKS_FROZEN) {
  5185. case CPU_UP_PREPARE:
  5186. rq->calc_load_update = calc_load_update;
  5187. break;
  5188. case CPU_ONLINE:
  5189. /* Update our root-domain */
  5190. raw_spin_lock_irqsave(&rq->lock, flags);
  5191. if (rq->rd) {
  5192. BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
  5193. set_rq_online(rq);
  5194. }
  5195. raw_spin_unlock_irqrestore(&rq->lock, flags);
  5196. break;
  5197. #ifdef CONFIG_HOTPLUG_CPU
  5198. case CPU_DYING:
  5199. /* Update our root-domain */
  5200. raw_spin_lock_irqsave(&rq->lock, flags);
  5201. if (rq->rd) {
  5202. BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
  5203. set_rq_offline(rq);
  5204. }
  5205. migrate_tasks(cpu);
  5206. BUG_ON(rq->nr_running != 1); /* the migration thread */
  5207. raw_spin_unlock_irqrestore(&rq->lock, flags);
  5208. migrate_nr_uninterruptible(rq);
  5209. calc_global_load_remove(rq);
  5210. break;
  5211. #endif
  5212. }
  5213. return NOTIFY_OK;
  5214. }
  5215. /*
  5216. * Register at high priority so that task migration (migrate_all_tasks)
  5217. * happens before everything else. This has to be lower priority than
  5218. * the notifier in the perf_event subsystem, though.
  5219. */
  5220. static struct notifier_block __cpuinitdata migration_notifier = {
  5221. .notifier_call = migration_call,
  5222. .priority = CPU_PRI_MIGRATION,
  5223. };
  5224. static int __cpuinit sched_cpu_active(struct notifier_block *nfb,
  5225. unsigned long action, void *hcpu)
  5226. {
  5227. switch (action & ~CPU_TASKS_FROZEN) {
  5228. case CPU_ONLINE:
  5229. case CPU_DOWN_FAILED:
  5230. set_cpu_active((long)hcpu, true);
  5231. return NOTIFY_OK;
  5232. default:
  5233. return NOTIFY_DONE;
  5234. }
  5235. }
  5236. static int __cpuinit sched_cpu_inactive(struct notifier_block *nfb,
  5237. unsigned long action, void *hcpu)
  5238. {
  5239. switch (action & ~CPU_TASKS_FROZEN) {
  5240. case CPU_DOWN_PREPARE:
  5241. set_cpu_active((long)hcpu, false);
  5242. return NOTIFY_OK;
  5243. default:
  5244. return NOTIFY_DONE;
  5245. }
  5246. }
  5247. static int __init migration_init(void)
  5248. {
  5249. void *cpu = (void *)(long)smp_processor_id();
  5250. int err;
  5251. /* Initialize migration for the boot CPU */
  5252. err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
  5253. BUG_ON(err == NOTIFY_BAD);
  5254. migration_call(&migration_notifier, CPU_ONLINE, cpu);
  5255. register_cpu_notifier(&migration_notifier);
  5256. /* Register cpu active notifiers */
  5257. cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
  5258. cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
  5259. return 0;
  5260. }
  5261. early_initcall(migration_init);
  5262. #endif
  5263. #ifdef CONFIG_SMP
  5264. #ifdef CONFIG_SCHED_DEBUG
  5265. static __read_mostly int sched_domain_debug_enabled;
  5266. static int __init sched_domain_debug_setup(char *str)
  5267. {
  5268. sched_domain_debug_enabled = 1;
  5269. return 0;
  5270. }
  5271. early_param("sched_debug", sched_domain_debug_setup);
  5272. static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
  5273. struct cpumask *groupmask)
  5274. {
  5275. struct sched_group *group = sd->groups;
  5276. char str[256];
  5277. cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
  5278. cpumask_clear(groupmask);
  5279. printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
  5280. if (!(sd->flags & SD_LOAD_BALANCE)) {
  5281. printk("does not load-balance\n");
  5282. if (sd->parent)
  5283. printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
  5284. " has parent");
  5285. return -1;
  5286. }
  5287. printk(KERN_CONT "span %s level %s\n", str, sd->name);
  5288. if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
  5289. printk(KERN_ERR "ERROR: domain->span does not contain "
  5290. "CPU%d\n", cpu);
  5291. }
  5292. if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
  5293. printk(KERN_ERR "ERROR: domain->groups does not contain"
  5294. " CPU%d\n", cpu);
  5295. }
  5296. printk(KERN_DEBUG "%*s groups:", level + 1, "");
  5297. do {
  5298. if (!group) {
  5299. printk("\n");
  5300. printk(KERN_ERR "ERROR: group is NULL\n");
  5301. break;
  5302. }
  5303. if (!group->cpu_power) {
  5304. printk(KERN_CONT "\n");
  5305. printk(KERN_ERR "ERROR: domain->cpu_power not "
  5306. "set\n");
  5307. break;
  5308. }
  5309. if (!cpumask_weight(sched_group_cpus(group))) {
  5310. printk(KERN_CONT "\n");
  5311. printk(KERN_ERR "ERROR: empty group\n");
  5312. break;
  5313. }
  5314. if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
  5315. printk(KERN_CONT "\n");
  5316. printk(KERN_ERR "ERROR: repeated CPUs\n");
  5317. break;
  5318. }
  5319. cpumask_or(groupmask, groupmask, sched_group_cpus(group));
  5320. cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
  5321. printk(KERN_CONT " %s", str);
  5322. if (group->cpu_power != SCHED_LOAD_SCALE) {
  5323. printk(KERN_CONT " (cpu_power = %d)",
  5324. group->cpu_power);
  5325. }
  5326. group = group->next;
  5327. } while (group != sd->groups);
  5328. printk(KERN_CONT "\n");
  5329. if (!cpumask_equal(sched_domain_span(sd), groupmask))
  5330. printk(KERN_ERR "ERROR: groups don't span domain->span\n");
  5331. if (sd->parent &&
  5332. !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
  5333. printk(KERN_ERR "ERROR: parent span is not a superset "
  5334. "of domain->span\n");
  5335. return 0;
  5336. }
  5337. static void sched_domain_debug(struct sched_domain *sd, int cpu)
  5338. {
  5339. cpumask_var_t groupmask;
  5340. int level = 0;
  5341. if (!sched_domain_debug_enabled)
  5342. return;
  5343. if (!sd) {
  5344. printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
  5345. return;
  5346. }
  5347. printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
  5348. if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
  5349. printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
  5350. return;
  5351. }
  5352. for (;;) {
  5353. if (sched_domain_debug_one(sd, cpu, level, groupmask))
  5354. break;
  5355. level++;
  5356. sd = sd->parent;
  5357. if (!sd)
  5358. break;
  5359. }
  5360. free_cpumask_var(groupmask);
  5361. }
  5362. #else /* !CONFIG_SCHED_DEBUG */
  5363. # define sched_domain_debug(sd, cpu) do { } while (0)
  5364. #endif /* CONFIG_SCHED_DEBUG */
  5365. static int sd_degenerate(struct sched_domain *sd)
  5366. {
  5367. if (cpumask_weight(sched_domain_span(sd)) == 1)
  5368. return 1;
  5369. /* Following flags need at least 2 groups */
  5370. if (sd->flags & (SD_LOAD_BALANCE |
  5371. SD_BALANCE_NEWIDLE |
  5372. SD_BALANCE_FORK |
  5373. SD_BALANCE_EXEC |
  5374. SD_SHARE_CPUPOWER |
  5375. SD_SHARE_PKG_RESOURCES)) {
  5376. if (sd->groups != sd->groups->next)
  5377. return 0;
  5378. }
  5379. /* Following flags don't use groups */
  5380. if (sd->flags & (SD_WAKE_AFFINE))
  5381. return 0;
  5382. return 1;
  5383. }
  5384. static int
  5385. sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
  5386. {
  5387. unsigned long cflags = sd->flags, pflags = parent->flags;
  5388. if (sd_degenerate(parent))
  5389. return 1;
  5390. if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
  5391. return 0;
  5392. /* Flags needing groups don't count if only 1 group in parent */
  5393. if (parent->groups == parent->groups->next) {
  5394. pflags &= ~(SD_LOAD_BALANCE |
  5395. SD_BALANCE_NEWIDLE |
  5396. SD_BALANCE_FORK |
  5397. SD_BALANCE_EXEC |
  5398. SD_SHARE_CPUPOWER |
  5399. SD_SHARE_PKG_RESOURCES);
  5400. if (nr_node_ids == 1)
  5401. pflags &= ~SD_SERIALIZE;
  5402. }
  5403. if (~cflags & pflags)
  5404. return 0;
  5405. return 1;
  5406. }
  5407. static void free_rootdomain(struct root_domain *rd)
  5408. {
  5409. synchronize_sched();
  5410. cpupri_cleanup(&rd->cpupri);
  5411. free_cpumask_var(rd->rto_mask);
  5412. free_cpumask_var(rd->online);
  5413. free_cpumask_var(rd->span);
  5414. kfree(rd);
  5415. }
  5416. static void rq_attach_root(struct rq *rq, struct root_domain *rd)
  5417. {
  5418. struct root_domain *old_rd = NULL;
  5419. unsigned long flags;
  5420. raw_spin_lock_irqsave(&rq->lock, flags);
  5421. if (rq->rd) {
  5422. old_rd = rq->rd;
  5423. if (cpumask_test_cpu(rq->cpu, old_rd->online))
  5424. set_rq_offline(rq);
  5425. cpumask_clear_cpu(rq->cpu, old_rd->span);
  5426. /*
  5427. * If we dont want to free the old_rt yet then
  5428. * set old_rd to NULL to skip the freeing later
  5429. * in this function:
  5430. */
  5431. if (!atomic_dec_and_test(&old_rd->refcount))
  5432. old_rd = NULL;
  5433. }
  5434. atomic_inc(&rd->refcount);
  5435. rq->rd = rd;
  5436. cpumask_set_cpu(rq->cpu, rd->span);
  5437. if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
  5438. set_rq_online(rq);
  5439. raw_spin_unlock_irqrestore(&rq->lock, flags);
  5440. if (old_rd)
  5441. free_rootdomain(old_rd);
  5442. }
  5443. static int init_rootdomain(struct root_domain *rd)
  5444. {
  5445. memset(rd, 0, sizeof(*rd));
  5446. if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
  5447. goto out;
  5448. if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
  5449. goto free_span;
  5450. if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
  5451. goto free_online;
  5452. if (cpupri_init(&rd->cpupri) != 0)
  5453. goto free_rto_mask;
  5454. return 0;
  5455. free_rto_mask:
  5456. free_cpumask_var(rd->rto_mask);
  5457. free_online:
  5458. free_cpumask_var(rd->online);
  5459. free_span:
  5460. free_cpumask_var(rd->span);
  5461. out:
  5462. return -ENOMEM;
  5463. }
  5464. static void init_defrootdomain(void)
  5465. {
  5466. init_rootdomain(&def_root_domain);
  5467. atomic_set(&def_root_domain.refcount, 1);
  5468. }
  5469. static struct root_domain *alloc_rootdomain(void)
  5470. {
  5471. struct root_domain *rd;
  5472. rd = kmalloc(sizeof(*rd), GFP_KERNEL);
  5473. if (!rd)
  5474. return NULL;
  5475. if (init_rootdomain(rd) != 0) {
  5476. kfree(rd);
  5477. return NULL;
  5478. }
  5479. return rd;
  5480. }
  5481. /*
  5482. * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
  5483. * hold the hotplug lock.
  5484. */
  5485. static void
  5486. cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
  5487. {
  5488. struct rq *rq = cpu_rq(cpu);
  5489. struct sched_domain *tmp;
  5490. for (tmp = sd; tmp; tmp = tmp->parent)
  5491. tmp->span_weight = cpumask_weight(sched_domain_span(tmp));
  5492. /* Remove the sched domains which do not contribute to scheduling. */
  5493. for (tmp = sd; tmp; ) {
  5494. struct sched_domain *parent = tmp->parent;
  5495. if (!parent)
  5496. break;
  5497. if (sd_parent_degenerate(tmp, parent)) {
  5498. tmp->parent = parent->parent;
  5499. if (parent->parent)
  5500. parent->parent->child = tmp;
  5501. } else
  5502. tmp = tmp->parent;
  5503. }
  5504. if (sd && sd_degenerate(sd)) {
  5505. sd = sd->parent;
  5506. if (sd)
  5507. sd->child = NULL;
  5508. }
  5509. sched_domain_debug(sd, cpu);
  5510. rq_attach_root(rq, rd);
  5511. rcu_assign_pointer(rq->sd, sd);
  5512. }
  5513. /* cpus with isolated domains */
  5514. static cpumask_var_t cpu_isolated_map;
  5515. /* Setup the mask of cpus configured for isolated domains */
  5516. static int __init isolated_cpu_setup(char *str)
  5517. {
  5518. alloc_bootmem_cpumask_var(&cpu_isolated_map);
  5519. cpulist_parse(str, cpu_isolated_map);
  5520. return 1;
  5521. }
  5522. __setup("isolcpus=", isolated_cpu_setup);
  5523. /*
  5524. * init_sched_build_groups takes the cpumask we wish to span, and a pointer
  5525. * to a function which identifies what group(along with sched group) a CPU
  5526. * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
  5527. * (due to the fact that we keep track of groups covered with a struct cpumask).
  5528. *
  5529. * init_sched_build_groups will build a circular linked list of the groups
  5530. * covered by the given span, and will set each group's ->cpumask correctly,
  5531. * and ->cpu_power to 0.
  5532. */
  5533. static void
  5534. init_sched_build_groups(const struct cpumask *span,
  5535. const struct cpumask *cpu_map,
  5536. int (*group_fn)(int cpu, const struct cpumask *cpu_map,
  5537. struct sched_group **sg,
  5538. struct cpumask *tmpmask),
  5539. struct cpumask *covered, struct cpumask *tmpmask)
  5540. {
  5541. struct sched_group *first = NULL, *last = NULL;
  5542. int i;
  5543. cpumask_clear(covered);
  5544. for_each_cpu(i, span) {
  5545. struct sched_group *sg;
  5546. int group = group_fn(i, cpu_map, &sg, tmpmask);
  5547. int j;
  5548. if (cpumask_test_cpu(i, covered))
  5549. continue;
  5550. cpumask_clear(sched_group_cpus(sg));
  5551. sg->cpu_power = 0;
  5552. for_each_cpu(j, span) {
  5553. if (group_fn(j, cpu_map, NULL, tmpmask) != group)
  5554. continue;
  5555. cpumask_set_cpu(j, covered);
  5556. cpumask_set_cpu(j, sched_group_cpus(sg));
  5557. }
  5558. if (!first)
  5559. first = sg;
  5560. if (last)
  5561. last->next = sg;
  5562. last = sg;
  5563. }
  5564. last->next = first;
  5565. }
  5566. #define SD_NODES_PER_DOMAIN 16
  5567. #ifdef CONFIG_NUMA
  5568. /**
  5569. * find_next_best_node - find the next node to include in a sched_domain
  5570. * @node: node whose sched_domain we're building
  5571. * @used_nodes: nodes already in the sched_domain
  5572. *
  5573. * Find the next node to include in a given scheduling domain. Simply
  5574. * finds the closest node not already in the @used_nodes map.
  5575. *
  5576. * Should use nodemask_t.
  5577. */
  5578. static int find_next_best_node(int node, nodemask_t *used_nodes)
  5579. {
  5580. int i, n, val, min_val, best_node = 0;
  5581. min_val = INT_MAX;
  5582. for (i = 0; i < nr_node_ids; i++) {
  5583. /* Start at @node */
  5584. n = (node + i) % nr_node_ids;
  5585. if (!nr_cpus_node(n))
  5586. continue;
  5587. /* Skip already used nodes */
  5588. if (node_isset(n, *used_nodes))
  5589. continue;
  5590. /* Simple min distance search */
  5591. val = node_distance(node, n);
  5592. if (val < min_val) {
  5593. min_val = val;
  5594. best_node = n;
  5595. }
  5596. }
  5597. node_set(best_node, *used_nodes);
  5598. return best_node;
  5599. }
  5600. /**
  5601. * sched_domain_node_span - get a cpumask for a node's sched_domain
  5602. * @node: node whose cpumask we're constructing
  5603. * @span: resulting cpumask
  5604. *
  5605. * Given a node, construct a good cpumask for its sched_domain to span. It
  5606. * should be one that prevents unnecessary balancing, but also spreads tasks
  5607. * out optimally.
  5608. */
  5609. static void sched_domain_node_span(int node, struct cpumask *span)
  5610. {
  5611. nodemask_t used_nodes;
  5612. int i;
  5613. cpumask_clear(span);
  5614. nodes_clear(used_nodes);
  5615. cpumask_or(span, span, cpumask_of_node(node));
  5616. node_set(node, used_nodes);
  5617. for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
  5618. int next_node = find_next_best_node(node, &used_nodes);
  5619. cpumask_or(span, span, cpumask_of_node(next_node));
  5620. }
  5621. }
  5622. #endif /* CONFIG_NUMA */
  5623. int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
  5624. /*
  5625. * The cpus mask in sched_group and sched_domain hangs off the end.
  5626. *
  5627. * ( See the the comments in include/linux/sched.h:struct sched_group
  5628. * and struct sched_domain. )
  5629. */
  5630. struct static_sched_group {
  5631. struct sched_group sg;
  5632. DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
  5633. };
  5634. struct static_sched_domain {
  5635. struct sched_domain sd;
  5636. DECLARE_BITMAP(span, CONFIG_NR_CPUS);
  5637. };
  5638. struct s_data {
  5639. #ifdef CONFIG_NUMA
  5640. int sd_allnodes;
  5641. cpumask_var_t domainspan;
  5642. cpumask_var_t covered;
  5643. cpumask_var_t notcovered;
  5644. #endif
  5645. cpumask_var_t nodemask;
  5646. cpumask_var_t this_sibling_map;
  5647. cpumask_var_t this_core_map;
  5648. cpumask_var_t this_book_map;
  5649. cpumask_var_t send_covered;
  5650. cpumask_var_t tmpmask;
  5651. struct sched_group **sched_group_nodes;
  5652. struct root_domain *rd;
  5653. };
  5654. enum s_alloc {
  5655. sa_sched_groups = 0,
  5656. sa_rootdomain,
  5657. sa_tmpmask,
  5658. sa_send_covered,
  5659. sa_this_book_map,
  5660. sa_this_core_map,
  5661. sa_this_sibling_map,
  5662. sa_nodemask,
  5663. sa_sched_group_nodes,
  5664. #ifdef CONFIG_NUMA
  5665. sa_notcovered,
  5666. sa_covered,
  5667. sa_domainspan,
  5668. #endif
  5669. sa_none,
  5670. };
  5671. /*
  5672. * SMT sched-domains:
  5673. */
  5674. #ifdef CONFIG_SCHED_SMT
  5675. static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
  5676. static DEFINE_PER_CPU(struct static_sched_group, sched_groups);
  5677. static int
  5678. cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
  5679. struct sched_group **sg, struct cpumask *unused)
  5680. {
  5681. if (sg)
  5682. *sg = &per_cpu(sched_groups, cpu).sg;
  5683. return cpu;
  5684. }
  5685. #endif /* CONFIG_SCHED_SMT */
  5686. /*
  5687. * multi-core sched-domains:
  5688. */
  5689. #ifdef CONFIG_SCHED_MC
  5690. static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
  5691. static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
  5692. static int
  5693. cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
  5694. struct sched_group **sg, struct cpumask *mask)
  5695. {
  5696. int group;
  5697. #ifdef CONFIG_SCHED_SMT
  5698. cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
  5699. group = cpumask_first(mask);
  5700. #else
  5701. group = cpu;
  5702. #endif
  5703. if (sg)
  5704. *sg = &per_cpu(sched_group_core, group).sg;
  5705. return group;
  5706. }
  5707. #endif /* CONFIG_SCHED_MC */
  5708. /*
  5709. * book sched-domains:
  5710. */
  5711. #ifdef CONFIG_SCHED_BOOK
  5712. static DEFINE_PER_CPU(struct static_sched_domain, book_domains);
  5713. static DEFINE_PER_CPU(struct static_sched_group, sched_group_book);
  5714. static int
  5715. cpu_to_book_group(int cpu, const struct cpumask *cpu_map,
  5716. struct sched_group **sg, struct cpumask *mask)
  5717. {
  5718. int group = cpu;
  5719. #ifdef CONFIG_SCHED_MC
  5720. cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
  5721. group = cpumask_first(mask);
  5722. #elif defined(CONFIG_SCHED_SMT)
  5723. cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
  5724. group = cpumask_first(mask);
  5725. #endif
  5726. if (sg)
  5727. *sg = &per_cpu(sched_group_book, group).sg;
  5728. return group;
  5729. }
  5730. #endif /* CONFIG_SCHED_BOOK */
  5731. static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
  5732. static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
  5733. static int
  5734. cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
  5735. struct sched_group **sg, struct cpumask *mask)
  5736. {
  5737. int group;
  5738. #ifdef CONFIG_SCHED_BOOK
  5739. cpumask_and(mask, cpu_book_mask(cpu), cpu_map);
  5740. group = cpumask_first(mask);
  5741. #elif defined(CONFIG_SCHED_MC)
  5742. cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
  5743. group = cpumask_first(mask);
  5744. #elif defined(CONFIG_SCHED_SMT)
  5745. cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
  5746. group = cpumask_first(mask);
  5747. #else
  5748. group = cpu;
  5749. #endif
  5750. if (sg)
  5751. *sg = &per_cpu(sched_group_phys, group).sg;
  5752. return group;
  5753. }
  5754. #ifdef CONFIG_NUMA
  5755. /*
  5756. * The init_sched_build_groups can't handle what we want to do with node
  5757. * groups, so roll our own. Now each node has its own list of groups which
  5758. * gets dynamically allocated.
  5759. */
  5760. static DEFINE_PER_CPU(struct static_sched_domain, node_domains);
  5761. static struct sched_group ***sched_group_nodes_bycpu;
  5762. static DEFINE_PER_CPU(struct static_sched_domain, allnodes_domains);
  5763. static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
  5764. static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
  5765. struct sched_group **sg,
  5766. struct cpumask *nodemask)
  5767. {
  5768. int group;
  5769. cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
  5770. group = cpumask_first(nodemask);
  5771. if (sg)
  5772. *sg = &per_cpu(sched_group_allnodes, group).sg;
  5773. return group;
  5774. }
  5775. static void init_numa_sched_groups_power(struct sched_group *group_head)
  5776. {
  5777. struct sched_group *sg = group_head;
  5778. int j;
  5779. if (!sg)
  5780. return;
  5781. do {
  5782. for_each_cpu(j, sched_group_cpus(sg)) {
  5783. struct sched_domain *sd;
  5784. sd = &per_cpu(phys_domains, j).sd;
  5785. if (j != group_first_cpu(sd->groups)) {
  5786. /*
  5787. * Only add "power" once for each
  5788. * physical package.
  5789. */
  5790. continue;
  5791. }
  5792. sg->cpu_power += sd->groups->cpu_power;
  5793. }
  5794. sg = sg->next;
  5795. } while (sg != group_head);
  5796. }
  5797. static int build_numa_sched_groups(struct s_data *d,
  5798. const struct cpumask *cpu_map, int num)
  5799. {
  5800. struct sched_domain *sd;
  5801. struct sched_group *sg, *prev;
  5802. int n, j;
  5803. cpumask_clear(d->covered);
  5804. cpumask_and(d->nodemask, cpumask_of_node(num), cpu_map);
  5805. if (cpumask_empty(d->nodemask)) {
  5806. d->sched_group_nodes[num] = NULL;
  5807. goto out;
  5808. }
  5809. sched_domain_node_span(num, d->domainspan);
  5810. cpumask_and(d->domainspan, d->domainspan, cpu_map);
  5811. sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
  5812. GFP_KERNEL, num);
  5813. if (!sg) {
  5814. printk(KERN_WARNING "Can not alloc domain group for node %d\n",
  5815. num);
  5816. return -ENOMEM;
  5817. }
  5818. d->sched_group_nodes[num] = sg;
  5819. for_each_cpu(j, d->nodemask) {
  5820. sd = &per_cpu(node_domains, j).sd;
  5821. sd->groups = sg;
  5822. }
  5823. sg->cpu_power = 0;
  5824. cpumask_copy(sched_group_cpus(sg), d->nodemask);
  5825. sg->next = sg;
  5826. cpumask_or(d->covered, d->covered, d->nodemask);
  5827. prev = sg;
  5828. for (j = 0; j < nr_node_ids; j++) {
  5829. n = (num + j) % nr_node_ids;
  5830. cpumask_complement(d->notcovered, d->covered);
  5831. cpumask_and(d->tmpmask, d->notcovered, cpu_map);
  5832. cpumask_and(d->tmpmask, d->tmpmask, d->domainspan);
  5833. if (cpumask_empty(d->tmpmask))
  5834. break;
  5835. cpumask_and(d->tmpmask, d->tmpmask, cpumask_of_node(n));
  5836. if (cpumask_empty(d->tmpmask))
  5837. continue;
  5838. sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
  5839. GFP_KERNEL, num);
  5840. if (!sg) {
  5841. printk(KERN_WARNING
  5842. "Can not alloc domain group for node %d\n", j);
  5843. return -ENOMEM;
  5844. }
  5845. sg->cpu_power = 0;
  5846. cpumask_copy(sched_group_cpus(sg), d->tmpmask);
  5847. sg->next = prev->next;
  5848. cpumask_or(d->covered, d->covered, d->tmpmask);
  5849. prev->next = sg;
  5850. prev = sg;
  5851. }
  5852. out:
  5853. return 0;
  5854. }
  5855. #endif /* CONFIG_NUMA */
  5856. #ifdef CONFIG_NUMA
  5857. /* Free memory allocated for various sched_group structures */
  5858. static void free_sched_groups(const struct cpumask *cpu_map,
  5859. struct cpumask *nodemask)
  5860. {
  5861. int cpu, i;
  5862. for_each_cpu(cpu, cpu_map) {
  5863. struct sched_group **sched_group_nodes
  5864. = sched_group_nodes_bycpu[cpu];
  5865. if (!sched_group_nodes)
  5866. continue;
  5867. for (i = 0; i < nr_node_ids; i++) {
  5868. struct sched_group *oldsg, *sg = sched_group_nodes[i];
  5869. cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
  5870. if (cpumask_empty(nodemask))
  5871. continue;
  5872. if (sg == NULL)
  5873. continue;
  5874. sg = sg->next;
  5875. next_sg:
  5876. oldsg = sg;
  5877. sg = sg->next;
  5878. kfree(oldsg);
  5879. if (oldsg != sched_group_nodes[i])
  5880. goto next_sg;
  5881. }
  5882. kfree(sched_group_nodes);
  5883. sched_group_nodes_bycpu[cpu] = NULL;
  5884. }
  5885. }
  5886. #else /* !CONFIG_NUMA */
  5887. static void free_sched_groups(const struct cpumask *cpu_map,
  5888. struct cpumask *nodemask)
  5889. {
  5890. }
  5891. #endif /* CONFIG_NUMA */
  5892. /*
  5893. * Initialize sched groups cpu_power.
  5894. *
  5895. * cpu_power indicates the capacity of sched group, which is used while
  5896. * distributing the load between different sched groups in a sched domain.
  5897. * Typically cpu_power for all the groups in a sched domain will be same unless
  5898. * there are asymmetries in the topology. If there are asymmetries, group
  5899. * having more cpu_power will pickup more load compared to the group having
  5900. * less cpu_power.
  5901. */
  5902. static void init_sched_groups_power(int cpu, struct sched_domain *sd)
  5903. {
  5904. struct sched_domain *child;
  5905. struct sched_group *group;
  5906. long power;
  5907. int weight;
  5908. WARN_ON(!sd || !sd->groups);
  5909. if (cpu != group_first_cpu(sd->groups))
  5910. return;
  5911. sd->groups->group_weight = cpumask_weight(sched_group_cpus(sd->groups));
  5912. child = sd->child;
  5913. sd->groups->cpu_power = 0;
  5914. if (!child) {
  5915. power = SCHED_LOAD_SCALE;
  5916. weight = cpumask_weight(sched_domain_span(sd));
  5917. /*
  5918. * SMT siblings share the power of a single core.
  5919. * Usually multiple threads get a better yield out of
  5920. * that one core than a single thread would have,
  5921. * reflect that in sd->smt_gain.
  5922. */
  5923. if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
  5924. power *= sd->smt_gain;
  5925. power /= weight;
  5926. power >>= SCHED_LOAD_SHIFT;
  5927. }
  5928. sd->groups->cpu_power += power;
  5929. return;
  5930. }
  5931. /*
  5932. * Add cpu_power of each child group to this groups cpu_power.
  5933. */
  5934. group = child->groups;
  5935. do {
  5936. sd->groups->cpu_power += group->cpu_power;
  5937. group = group->next;
  5938. } while (group != child->groups);
  5939. }
  5940. /*
  5941. * Initializers for schedule domains
  5942. * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
  5943. */
  5944. #ifdef CONFIG_SCHED_DEBUG
  5945. # define SD_INIT_NAME(sd, type) sd->name = #type
  5946. #else
  5947. # define SD_INIT_NAME(sd, type) do { } while (0)
  5948. #endif
  5949. #define SD_INIT(sd, type) sd_init_##type(sd)
  5950. #define SD_INIT_FUNC(type) \
  5951. static noinline void sd_init_##type(struct sched_domain *sd) \
  5952. { \
  5953. memset(sd, 0, sizeof(*sd)); \
  5954. *sd = SD_##type##_INIT; \
  5955. sd->level = SD_LV_##type; \
  5956. SD_INIT_NAME(sd, type); \
  5957. }
  5958. SD_INIT_FUNC(CPU)
  5959. #ifdef CONFIG_NUMA
  5960. SD_INIT_FUNC(ALLNODES)
  5961. SD_INIT_FUNC(NODE)
  5962. #endif
  5963. #ifdef CONFIG_SCHED_SMT
  5964. SD_INIT_FUNC(SIBLING)
  5965. #endif
  5966. #ifdef CONFIG_SCHED_MC
  5967. SD_INIT_FUNC(MC)
  5968. #endif
  5969. #ifdef CONFIG_SCHED_BOOK
  5970. SD_INIT_FUNC(BOOK)
  5971. #endif
  5972. static int default_relax_domain_level = -1;
  5973. static int __init setup_relax_domain_level(char *str)
  5974. {
  5975. unsigned long val;
  5976. val = simple_strtoul(str, NULL, 0);
  5977. if (val < SD_LV_MAX)
  5978. default_relax_domain_level = val;
  5979. return 1;
  5980. }
  5981. __setup("relax_domain_level=", setup_relax_domain_level);
  5982. static void set_domain_attribute(struct sched_domain *sd,
  5983. struct sched_domain_attr *attr)
  5984. {
  5985. int request;
  5986. if (!attr || attr->relax_domain_level < 0) {
  5987. if (default_relax_domain_level < 0)
  5988. return;
  5989. else
  5990. request = default_relax_domain_level;
  5991. } else
  5992. request = attr->relax_domain_level;
  5993. if (request < sd->level) {
  5994. /* turn off idle balance on this domain */
  5995. sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
  5996. } else {
  5997. /* turn on idle balance on this domain */
  5998. sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
  5999. }
  6000. }
  6001. static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
  6002. const struct cpumask *cpu_map)
  6003. {
  6004. switch (what) {
  6005. case sa_sched_groups:
  6006. free_sched_groups(cpu_map, d->tmpmask); /* fall through */
  6007. d->sched_group_nodes = NULL;
  6008. case sa_rootdomain:
  6009. free_rootdomain(d->rd); /* fall through */
  6010. case sa_tmpmask:
  6011. free_cpumask_var(d->tmpmask); /* fall through */
  6012. case sa_send_covered:
  6013. free_cpumask_var(d->send_covered); /* fall through */
  6014. case sa_this_book_map:
  6015. free_cpumask_var(d->this_book_map); /* fall through */
  6016. case sa_this_core_map:
  6017. free_cpumask_var(d->this_core_map); /* fall through */
  6018. case sa_this_sibling_map:
  6019. free_cpumask_var(d->this_sibling_map); /* fall through */
  6020. case sa_nodemask:
  6021. free_cpumask_var(d->nodemask); /* fall through */
  6022. case sa_sched_group_nodes:
  6023. #ifdef CONFIG_NUMA
  6024. kfree(d->sched_group_nodes); /* fall through */
  6025. case sa_notcovered:
  6026. free_cpumask_var(d->notcovered); /* fall through */
  6027. case sa_covered:
  6028. free_cpumask_var(d->covered); /* fall through */
  6029. case sa_domainspan:
  6030. free_cpumask_var(d->domainspan); /* fall through */
  6031. #endif
  6032. case sa_none:
  6033. break;
  6034. }
  6035. }
  6036. static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
  6037. const struct cpumask *cpu_map)
  6038. {
  6039. #ifdef CONFIG_NUMA
  6040. if (!alloc_cpumask_var(&d->domainspan, GFP_KERNEL))
  6041. return sa_none;
  6042. if (!alloc_cpumask_var(&d->covered, GFP_KERNEL))
  6043. return sa_domainspan;
  6044. if (!alloc_cpumask_var(&d->notcovered, GFP_KERNEL))
  6045. return sa_covered;
  6046. /* Allocate the per-node list of sched groups */
  6047. d->sched_group_nodes = kcalloc(nr_node_ids,
  6048. sizeof(struct sched_group *), GFP_KERNEL);
  6049. if (!d->sched_group_nodes) {
  6050. printk(KERN_WARNING "Can not alloc sched group node list\n");
  6051. return sa_notcovered;
  6052. }
  6053. sched_group_nodes_bycpu[cpumask_first(cpu_map)] = d->sched_group_nodes;
  6054. #endif
  6055. if (!alloc_cpumask_var(&d->nodemask, GFP_KERNEL))
  6056. return sa_sched_group_nodes;
  6057. if (!alloc_cpumask_var(&d->this_sibling_map, GFP_KERNEL))
  6058. return sa_nodemask;
  6059. if (!alloc_cpumask_var(&d->this_core_map, GFP_KERNEL))
  6060. return sa_this_sibling_map;
  6061. if (!alloc_cpumask_var(&d->this_book_map, GFP_KERNEL))
  6062. return sa_this_core_map;
  6063. if (!alloc_cpumask_var(&d->send_covered, GFP_KERNEL))
  6064. return sa_this_book_map;
  6065. if (!alloc_cpumask_var(&d->tmpmask, GFP_KERNEL))
  6066. return sa_send_covered;
  6067. d->rd = alloc_rootdomain();
  6068. if (!d->rd) {
  6069. printk(KERN_WARNING "Cannot alloc root domain\n");
  6070. return sa_tmpmask;
  6071. }
  6072. return sa_rootdomain;
  6073. }
  6074. static struct sched_domain *__build_numa_sched_domains(struct s_data *d,
  6075. const struct cpumask *cpu_map, struct sched_domain_attr *attr, int i)
  6076. {
  6077. struct sched_domain *sd = NULL;
  6078. #ifdef CONFIG_NUMA
  6079. struct sched_domain *parent;
  6080. d->sd_allnodes = 0;
  6081. if (cpumask_weight(cpu_map) >
  6082. SD_NODES_PER_DOMAIN * cpumask_weight(d->nodemask)) {
  6083. sd = &per_cpu(allnodes_domains, i).sd;
  6084. SD_INIT(sd, ALLNODES);
  6085. set_domain_attribute(sd, attr);
  6086. cpumask_copy(sched_domain_span(sd), cpu_map);
  6087. cpu_to_allnodes_group(i, cpu_map, &sd->groups, d->tmpmask);
  6088. d->sd_allnodes = 1;
  6089. }
  6090. parent = sd;
  6091. sd = &per_cpu(node_domains, i).sd;
  6092. SD_INIT(sd, NODE);
  6093. set_domain_attribute(sd, attr);
  6094. sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
  6095. sd->parent = parent;
  6096. if (parent)
  6097. parent->child = sd;
  6098. cpumask_and(sched_domain_span(sd), sched_domain_span(sd), cpu_map);
  6099. #endif
  6100. return sd;
  6101. }
  6102. static struct sched_domain *__build_cpu_sched_domain(struct s_data *d,
  6103. const struct cpumask *cpu_map, struct sched_domain_attr *attr,
  6104. struct sched_domain *parent, int i)
  6105. {
  6106. struct sched_domain *sd;
  6107. sd = &per_cpu(phys_domains, i).sd;
  6108. SD_INIT(sd, CPU);
  6109. set_domain_attribute(sd, attr);
  6110. cpumask_copy(sched_domain_span(sd), d->nodemask);
  6111. sd->parent = parent;
  6112. if (parent)
  6113. parent->child = sd;
  6114. cpu_to_phys_group(i, cpu_map, &sd->groups, d->tmpmask);
  6115. return sd;
  6116. }
  6117. static struct sched_domain *__build_book_sched_domain(struct s_data *d,
  6118. const struct cpumask *cpu_map, struct sched_domain_attr *attr,
  6119. struct sched_domain *parent, int i)
  6120. {
  6121. struct sched_domain *sd = parent;
  6122. #ifdef CONFIG_SCHED_BOOK
  6123. sd = &per_cpu(book_domains, i).sd;
  6124. SD_INIT(sd, BOOK);
  6125. set_domain_attribute(sd, attr);
  6126. cpumask_and(sched_domain_span(sd), cpu_map, cpu_book_mask(i));
  6127. sd->parent = parent;
  6128. parent->child = sd;
  6129. cpu_to_book_group(i, cpu_map, &sd->groups, d->tmpmask);
  6130. #endif
  6131. return sd;
  6132. }
  6133. static struct sched_domain *__build_mc_sched_domain(struct s_data *d,
  6134. const struct cpumask *cpu_map, struct sched_domain_attr *attr,
  6135. struct sched_domain *parent, int i)
  6136. {
  6137. struct sched_domain *sd = parent;
  6138. #ifdef CONFIG_SCHED_MC
  6139. sd = &per_cpu(core_domains, i).sd;
  6140. SD_INIT(sd, MC);
  6141. set_domain_attribute(sd, attr);
  6142. cpumask_and(sched_domain_span(sd), cpu_map, cpu_coregroup_mask(i));
  6143. sd->parent = parent;
  6144. parent->child = sd;
  6145. cpu_to_core_group(i, cpu_map, &sd->groups, d->tmpmask);
  6146. #endif
  6147. return sd;
  6148. }
  6149. static struct sched_domain *__build_smt_sched_domain(struct s_data *d,
  6150. const struct cpumask *cpu_map, struct sched_domain_attr *attr,
  6151. struct sched_domain *parent, int i)
  6152. {
  6153. struct sched_domain *sd = parent;
  6154. #ifdef CONFIG_SCHED_SMT
  6155. sd = &per_cpu(cpu_domains, i).sd;
  6156. SD_INIT(sd, SIBLING);
  6157. set_domain_attribute(sd, attr);
  6158. cpumask_and(sched_domain_span(sd), cpu_map, topology_thread_cpumask(i));
  6159. sd->parent = parent;
  6160. parent->child = sd;
  6161. cpu_to_cpu_group(i, cpu_map, &sd->groups, d->tmpmask);
  6162. #endif
  6163. return sd;
  6164. }
  6165. static void build_sched_groups(struct s_data *d, enum sched_domain_level l,
  6166. const struct cpumask *cpu_map, int cpu)
  6167. {
  6168. switch (l) {
  6169. #ifdef CONFIG_SCHED_SMT
  6170. case SD_LV_SIBLING: /* set up CPU (sibling) groups */
  6171. cpumask_and(d->this_sibling_map, cpu_map,
  6172. topology_thread_cpumask(cpu));
  6173. if (cpu == cpumask_first(d->this_sibling_map))
  6174. init_sched_build_groups(d->this_sibling_map, cpu_map,
  6175. &cpu_to_cpu_group,
  6176. d->send_covered, d->tmpmask);
  6177. break;
  6178. #endif
  6179. #ifdef CONFIG_SCHED_MC
  6180. case SD_LV_MC: /* set up multi-core groups */
  6181. cpumask_and(d->this_core_map, cpu_map, cpu_coregroup_mask(cpu));
  6182. if (cpu == cpumask_first(d->this_core_map))
  6183. init_sched_build_groups(d->this_core_map, cpu_map,
  6184. &cpu_to_core_group,
  6185. d->send_covered, d->tmpmask);
  6186. break;
  6187. #endif
  6188. #ifdef CONFIG_SCHED_BOOK
  6189. case SD_LV_BOOK: /* set up book groups */
  6190. cpumask_and(d->this_book_map, cpu_map, cpu_book_mask(cpu));
  6191. if (cpu == cpumask_first(d->this_book_map))
  6192. init_sched_build_groups(d->this_book_map, cpu_map,
  6193. &cpu_to_book_group,
  6194. d->send_covered, d->tmpmask);
  6195. break;
  6196. #endif
  6197. case SD_LV_CPU: /* set up physical groups */
  6198. cpumask_and(d->nodemask, cpumask_of_node(cpu), cpu_map);
  6199. if (!cpumask_empty(d->nodemask))
  6200. init_sched_build_groups(d->nodemask, cpu_map,
  6201. &cpu_to_phys_group,
  6202. d->send_covered, d->tmpmask);
  6203. break;
  6204. #ifdef CONFIG_NUMA
  6205. case SD_LV_ALLNODES:
  6206. init_sched_build_groups(cpu_map, cpu_map, &cpu_to_allnodes_group,
  6207. d->send_covered, d->tmpmask);
  6208. break;
  6209. #endif
  6210. default:
  6211. break;
  6212. }
  6213. }
  6214. /*
  6215. * Build sched domains for a given set of cpus and attach the sched domains
  6216. * to the individual cpus
  6217. */
  6218. static int __build_sched_domains(const struct cpumask *cpu_map,
  6219. struct sched_domain_attr *attr)
  6220. {
  6221. enum s_alloc alloc_state = sa_none;
  6222. struct s_data d;
  6223. struct sched_domain *sd;
  6224. int i;
  6225. #ifdef CONFIG_NUMA
  6226. d.sd_allnodes = 0;
  6227. #endif
  6228. alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
  6229. if (alloc_state != sa_rootdomain)
  6230. goto error;
  6231. alloc_state = sa_sched_groups;
  6232. /*
  6233. * Set up domains for cpus specified by the cpu_map.
  6234. */
  6235. for_each_cpu(i, cpu_map) {
  6236. cpumask_and(d.nodemask, cpumask_of_node(cpu_to_node(i)),
  6237. cpu_map);
  6238. sd = __build_numa_sched_domains(&d, cpu_map, attr, i);
  6239. sd = __build_cpu_sched_domain(&d, cpu_map, attr, sd, i);
  6240. sd = __build_book_sched_domain(&d, cpu_map, attr, sd, i);
  6241. sd = __build_mc_sched_domain(&d, cpu_map, attr, sd, i);
  6242. sd = __build_smt_sched_domain(&d, cpu_map, attr, sd, i);
  6243. }
  6244. for_each_cpu(i, cpu_map) {
  6245. build_sched_groups(&d, SD_LV_SIBLING, cpu_map, i);
  6246. build_sched_groups(&d, SD_LV_BOOK, cpu_map, i);
  6247. build_sched_groups(&d, SD_LV_MC, cpu_map, i);
  6248. }
  6249. /* Set up physical groups */
  6250. for (i = 0; i < nr_node_ids; i++)
  6251. build_sched_groups(&d, SD_LV_CPU, cpu_map, i);
  6252. #ifdef CONFIG_NUMA
  6253. /* Set up node groups */
  6254. if (d.sd_allnodes)
  6255. build_sched_groups(&d, SD_LV_ALLNODES, cpu_map, 0);
  6256. for (i = 0; i < nr_node_ids; i++)
  6257. if (build_numa_sched_groups(&d, cpu_map, i))
  6258. goto error;
  6259. #endif
  6260. /* Calculate CPU power for physical packages and nodes */
  6261. #ifdef CONFIG_SCHED_SMT
  6262. for_each_cpu(i, cpu_map) {
  6263. sd = &per_cpu(cpu_domains, i).sd;
  6264. init_sched_groups_power(i, sd);
  6265. }
  6266. #endif
  6267. #ifdef CONFIG_SCHED_MC
  6268. for_each_cpu(i, cpu_map) {
  6269. sd = &per_cpu(core_domains, i).sd;
  6270. init_sched_groups_power(i, sd);
  6271. }
  6272. #endif
  6273. #ifdef CONFIG_SCHED_BOOK
  6274. for_each_cpu(i, cpu_map) {
  6275. sd = &per_cpu(book_domains, i).sd;
  6276. init_sched_groups_power(i, sd);
  6277. }
  6278. #endif
  6279. for_each_cpu(i, cpu_map) {
  6280. sd = &per_cpu(phys_domains, i).sd;
  6281. init_sched_groups_power(i, sd);
  6282. }
  6283. #ifdef CONFIG_NUMA
  6284. for (i = 0; i < nr_node_ids; i++)
  6285. init_numa_sched_groups_power(d.sched_group_nodes[i]);
  6286. if (d.sd_allnodes) {
  6287. struct sched_group *sg;
  6288. cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
  6289. d.tmpmask);
  6290. init_numa_sched_groups_power(sg);
  6291. }
  6292. #endif
  6293. /* Attach the domains */
  6294. for_each_cpu(i, cpu_map) {
  6295. #ifdef CONFIG_SCHED_SMT
  6296. sd = &per_cpu(cpu_domains, i).sd;
  6297. #elif defined(CONFIG_SCHED_MC)
  6298. sd = &per_cpu(core_domains, i).sd;
  6299. #elif defined(CONFIG_SCHED_BOOK)
  6300. sd = &per_cpu(book_domains, i).sd;
  6301. #else
  6302. sd = &per_cpu(phys_domains, i).sd;
  6303. #endif
  6304. cpu_attach_domain(sd, d.rd, i);
  6305. }
  6306. d.sched_group_nodes = NULL; /* don't free this we still need it */
  6307. __free_domain_allocs(&d, sa_tmpmask, cpu_map);
  6308. return 0;
  6309. error:
  6310. __free_domain_allocs(&d, alloc_state, cpu_map);
  6311. return -ENOMEM;
  6312. }
  6313. static int build_sched_domains(const struct cpumask *cpu_map)
  6314. {
  6315. return __build_sched_domains(cpu_map, NULL);
  6316. }
  6317. static cpumask_var_t *doms_cur; /* current sched domains */
  6318. static int ndoms_cur; /* number of sched domains in 'doms_cur' */
  6319. static struct sched_domain_attr *dattr_cur;
  6320. /* attribues of custom domains in 'doms_cur' */
  6321. /*
  6322. * Special case: If a kmalloc of a doms_cur partition (array of
  6323. * cpumask) fails, then fallback to a single sched domain,
  6324. * as determined by the single cpumask fallback_doms.
  6325. */
  6326. static cpumask_var_t fallback_doms;
  6327. /*
  6328. * arch_update_cpu_topology lets virtualized architectures update the
  6329. * cpu core maps. It is supposed to return 1 if the topology changed
  6330. * or 0 if it stayed the same.
  6331. */
  6332. int __attribute__((weak)) arch_update_cpu_topology(void)
  6333. {
  6334. return 0;
  6335. }
  6336. cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
  6337. {
  6338. int i;
  6339. cpumask_var_t *doms;
  6340. doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
  6341. if (!doms)
  6342. return NULL;
  6343. for (i = 0; i < ndoms; i++) {
  6344. if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
  6345. free_sched_domains(doms, i);
  6346. return NULL;
  6347. }
  6348. }
  6349. return doms;
  6350. }
  6351. void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
  6352. {
  6353. unsigned int i;
  6354. for (i = 0; i < ndoms; i++)
  6355. free_cpumask_var(doms[i]);
  6356. kfree(doms);
  6357. }
  6358. /*
  6359. * Set up scheduler domains and groups. Callers must hold the hotplug lock.
  6360. * For now this just excludes isolated cpus, but could be used to
  6361. * exclude other special cases in the future.
  6362. */
  6363. static int arch_init_sched_domains(const struct cpumask *cpu_map)
  6364. {
  6365. int err;
  6366. arch_update_cpu_topology();
  6367. ndoms_cur = 1;
  6368. doms_cur = alloc_sched_domains(ndoms_cur);
  6369. if (!doms_cur)
  6370. doms_cur = &fallback_doms;
  6371. cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
  6372. dattr_cur = NULL;
  6373. err = build_sched_domains(doms_cur[0]);
  6374. register_sched_domain_sysctl();
  6375. return err;
  6376. }
  6377. static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
  6378. struct cpumask *tmpmask)
  6379. {
  6380. free_sched_groups(cpu_map, tmpmask);
  6381. }
  6382. /*
  6383. * Detach sched domains from a group of cpus specified in cpu_map
  6384. * These cpus will now be attached to the NULL domain
  6385. */
  6386. static void detach_destroy_domains(const struct cpumask *cpu_map)
  6387. {
  6388. /* Save because hotplug lock held. */
  6389. static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
  6390. int i;
  6391. for_each_cpu(i, cpu_map)
  6392. cpu_attach_domain(NULL, &def_root_domain, i);
  6393. synchronize_sched();
  6394. arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
  6395. }
  6396. /* handle null as "default" */
  6397. static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
  6398. struct sched_domain_attr *new, int idx_new)
  6399. {
  6400. struct sched_domain_attr tmp;
  6401. /* fast path */
  6402. if (!new && !cur)
  6403. return 1;
  6404. tmp = SD_ATTR_INIT;
  6405. return !memcmp(cur ? (cur + idx_cur) : &tmp,
  6406. new ? (new + idx_new) : &tmp,
  6407. sizeof(struct sched_domain_attr));
  6408. }
  6409. /*
  6410. * Partition sched domains as specified by the 'ndoms_new'
  6411. * cpumasks in the array doms_new[] of cpumasks. This compares
  6412. * doms_new[] to the current sched domain partitioning, doms_cur[].
  6413. * It destroys each deleted domain and builds each new domain.
  6414. *
  6415. * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
  6416. * The masks don't intersect (don't overlap.) We should setup one
  6417. * sched domain for each mask. CPUs not in any of the cpumasks will
  6418. * not be load balanced. If the same cpumask appears both in the
  6419. * current 'doms_cur' domains and in the new 'doms_new', we can leave
  6420. * it as it is.
  6421. *
  6422. * The passed in 'doms_new' should be allocated using
  6423. * alloc_sched_domains. This routine takes ownership of it and will
  6424. * free_sched_domains it when done with it. If the caller failed the
  6425. * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
  6426. * and partition_sched_domains() will fallback to the single partition
  6427. * 'fallback_doms', it also forces the domains to be rebuilt.
  6428. *
  6429. * If doms_new == NULL it will be replaced with cpu_online_mask.
  6430. * ndoms_new == 0 is a special case for destroying existing domains,
  6431. * and it will not create the default domain.
  6432. *
  6433. * Call with hotplug lock held
  6434. */
  6435. void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
  6436. struct sched_domain_attr *dattr_new)
  6437. {
  6438. int i, j, n;
  6439. int new_topology;
  6440. mutex_lock(&sched_domains_mutex);
  6441. /* always unregister in case we don't destroy any domains */
  6442. unregister_sched_domain_sysctl();
  6443. /* Let architecture update cpu core mappings. */
  6444. new_topology = arch_update_cpu_topology();
  6445. n = doms_new ? ndoms_new : 0;
  6446. /* Destroy deleted domains */
  6447. for (i = 0; i < ndoms_cur; i++) {
  6448. for (j = 0; j < n && !new_topology; j++) {
  6449. if (cpumask_equal(doms_cur[i], doms_new[j])
  6450. && dattrs_equal(dattr_cur, i, dattr_new, j))
  6451. goto match1;
  6452. }
  6453. /* no match - a current sched domain not in new doms_new[] */
  6454. detach_destroy_domains(doms_cur[i]);
  6455. match1:
  6456. ;
  6457. }
  6458. if (doms_new == NULL) {
  6459. ndoms_cur = 0;
  6460. doms_new = &fallback_doms;
  6461. cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
  6462. WARN_ON_ONCE(dattr_new);
  6463. }
  6464. /* Build new domains */
  6465. for (i = 0; i < ndoms_new; i++) {
  6466. for (j = 0; j < ndoms_cur && !new_topology; j++) {
  6467. if (cpumask_equal(doms_new[i], doms_cur[j])
  6468. && dattrs_equal(dattr_new, i, dattr_cur, j))
  6469. goto match2;
  6470. }
  6471. /* no match - add a new doms_new */
  6472. __build_sched_domains(doms_new[i],
  6473. dattr_new ? dattr_new + i : NULL);
  6474. match2:
  6475. ;
  6476. }
  6477. /* Remember the new sched domains */
  6478. if (doms_cur != &fallback_doms)
  6479. free_sched_domains(doms_cur, ndoms_cur);
  6480. kfree(dattr_cur); /* kfree(NULL) is safe */
  6481. doms_cur = doms_new;
  6482. dattr_cur = dattr_new;
  6483. ndoms_cur = ndoms_new;
  6484. register_sched_domain_sysctl();
  6485. mutex_unlock(&sched_domains_mutex);
  6486. }
  6487. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  6488. static void arch_reinit_sched_domains(void)
  6489. {
  6490. get_online_cpus();
  6491. /* Destroy domains first to force the rebuild */
  6492. partition_sched_domains(0, NULL, NULL);
  6493. rebuild_sched_domains();
  6494. put_online_cpus();
  6495. }
  6496. static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
  6497. {
  6498. unsigned int level = 0;
  6499. if (sscanf(buf, "%u", &level) != 1)
  6500. return -EINVAL;
  6501. /*
  6502. * level is always be positive so don't check for
  6503. * level < POWERSAVINGS_BALANCE_NONE which is 0
  6504. * What happens on 0 or 1 byte write,
  6505. * need to check for count as well?
  6506. */
  6507. if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
  6508. return -EINVAL;
  6509. if (smt)
  6510. sched_smt_power_savings = level;
  6511. else
  6512. sched_mc_power_savings = level;
  6513. arch_reinit_sched_domains();
  6514. return count;
  6515. }
  6516. #ifdef CONFIG_SCHED_MC
  6517. static ssize_t sched_mc_power_savings_show(struct sysdev_class *class,
  6518. struct sysdev_class_attribute *attr,
  6519. char *page)
  6520. {
  6521. return sprintf(page, "%u\n", sched_mc_power_savings);
  6522. }
  6523. static ssize_t sched_mc_power_savings_store(struct sysdev_class *class,
  6524. struct sysdev_class_attribute *attr,
  6525. const char *buf, size_t count)
  6526. {
  6527. return sched_power_savings_store(buf, count, 0);
  6528. }
  6529. static SYSDEV_CLASS_ATTR(sched_mc_power_savings, 0644,
  6530. sched_mc_power_savings_show,
  6531. sched_mc_power_savings_store);
  6532. #endif
  6533. #ifdef CONFIG_SCHED_SMT
  6534. static ssize_t sched_smt_power_savings_show(struct sysdev_class *dev,
  6535. struct sysdev_class_attribute *attr,
  6536. char *page)
  6537. {
  6538. return sprintf(page, "%u\n", sched_smt_power_savings);
  6539. }
  6540. static ssize_t sched_smt_power_savings_store(struct sysdev_class *dev,
  6541. struct sysdev_class_attribute *attr,
  6542. const char *buf, size_t count)
  6543. {
  6544. return sched_power_savings_store(buf, count, 1);
  6545. }
  6546. static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
  6547. sched_smt_power_savings_show,
  6548. sched_smt_power_savings_store);
  6549. #endif
  6550. int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
  6551. {
  6552. int err = 0;
  6553. #ifdef CONFIG_SCHED_SMT
  6554. if (smt_capable())
  6555. err = sysfs_create_file(&cls->kset.kobj,
  6556. &attr_sched_smt_power_savings.attr);
  6557. #endif
  6558. #ifdef CONFIG_SCHED_MC
  6559. if (!err && mc_capable())
  6560. err = sysfs_create_file(&cls->kset.kobj,
  6561. &attr_sched_mc_power_savings.attr);
  6562. #endif
  6563. return err;
  6564. }
  6565. #endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
  6566. /*
  6567. * Update cpusets according to cpu_active mask. If cpusets are
  6568. * disabled, cpuset_update_active_cpus() becomes a simple wrapper
  6569. * around partition_sched_domains().
  6570. */
  6571. static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
  6572. void *hcpu)
  6573. {
  6574. switch (action & ~CPU_TASKS_FROZEN) {
  6575. case CPU_ONLINE:
  6576. case CPU_DOWN_FAILED:
  6577. cpuset_update_active_cpus();
  6578. return NOTIFY_OK;
  6579. default:
  6580. return NOTIFY_DONE;
  6581. }
  6582. }
  6583. static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
  6584. void *hcpu)
  6585. {
  6586. switch (action & ~CPU_TASKS_FROZEN) {
  6587. case CPU_DOWN_PREPARE:
  6588. cpuset_update_active_cpus();
  6589. return NOTIFY_OK;
  6590. default:
  6591. return NOTIFY_DONE;
  6592. }
  6593. }
  6594. static int update_runtime(struct notifier_block *nfb,
  6595. unsigned long action, void *hcpu)
  6596. {
  6597. int cpu = (int)(long)hcpu;
  6598. switch (action) {
  6599. case CPU_DOWN_PREPARE:
  6600. case CPU_DOWN_PREPARE_FROZEN:
  6601. disable_runtime(cpu_rq(cpu));
  6602. return NOTIFY_OK;
  6603. case CPU_DOWN_FAILED:
  6604. case CPU_DOWN_FAILED_FROZEN:
  6605. case CPU_ONLINE:
  6606. case CPU_ONLINE_FROZEN:
  6607. enable_runtime(cpu_rq(cpu));
  6608. return NOTIFY_OK;
  6609. default:
  6610. return NOTIFY_DONE;
  6611. }
  6612. }
  6613. void __init sched_init_smp(void)
  6614. {
  6615. cpumask_var_t non_isolated_cpus;
  6616. alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
  6617. alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
  6618. #if defined(CONFIG_NUMA)
  6619. sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
  6620. GFP_KERNEL);
  6621. BUG_ON(sched_group_nodes_bycpu == NULL);
  6622. #endif
  6623. get_online_cpus();
  6624. mutex_lock(&sched_domains_mutex);
  6625. arch_init_sched_domains(cpu_active_mask);
  6626. cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
  6627. if (cpumask_empty(non_isolated_cpus))
  6628. cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
  6629. mutex_unlock(&sched_domains_mutex);
  6630. put_online_cpus();
  6631. hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
  6632. hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
  6633. /* RT runtime code needs to handle some hotplug events */
  6634. hotcpu_notifier(update_runtime, 0);
  6635. init_hrtick();
  6636. /* Move init over to a non-isolated CPU */
  6637. if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
  6638. BUG();
  6639. sched_init_granularity();
  6640. free_cpumask_var(non_isolated_cpus);
  6641. init_sched_rt_class();
  6642. }
  6643. #else
  6644. void __init sched_init_smp(void)
  6645. {
  6646. sched_init_granularity();
  6647. }
  6648. #endif /* CONFIG_SMP */
  6649. const_debug unsigned int sysctl_timer_migration = 1;
  6650. int in_sched_functions(unsigned long addr)
  6651. {
  6652. return in_lock_functions(addr) ||
  6653. (addr >= (unsigned long)__sched_text_start
  6654. && addr < (unsigned long)__sched_text_end);
  6655. }
  6656. static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
  6657. {
  6658. cfs_rq->tasks_timeline = RB_ROOT;
  6659. INIT_LIST_HEAD(&cfs_rq->tasks);
  6660. #ifdef CONFIG_FAIR_GROUP_SCHED
  6661. cfs_rq->rq = rq;
  6662. #endif
  6663. cfs_rq->min_vruntime = (u64)(-(1LL << 20));
  6664. }
  6665. static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
  6666. {
  6667. struct rt_prio_array *array;
  6668. int i;
  6669. array = &rt_rq->active;
  6670. for (i = 0; i < MAX_RT_PRIO; i++) {
  6671. INIT_LIST_HEAD(array->queue + i);
  6672. __clear_bit(i, array->bitmap);
  6673. }
  6674. /* delimiter for bitsearch: */
  6675. __set_bit(MAX_RT_PRIO, array->bitmap);
  6676. #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
  6677. rt_rq->highest_prio.curr = MAX_RT_PRIO;
  6678. #ifdef CONFIG_SMP
  6679. rt_rq->highest_prio.next = MAX_RT_PRIO;
  6680. #endif
  6681. #endif
  6682. #ifdef CONFIG_SMP
  6683. rt_rq->rt_nr_migratory = 0;
  6684. rt_rq->overloaded = 0;
  6685. plist_head_init_raw(&rt_rq->pushable_tasks, &rq->lock);
  6686. #endif
  6687. rt_rq->rt_time = 0;
  6688. rt_rq->rt_throttled = 0;
  6689. rt_rq->rt_runtime = 0;
  6690. raw_spin_lock_init(&rt_rq->rt_runtime_lock);
  6691. #ifdef CONFIG_RT_GROUP_SCHED
  6692. rt_rq->rt_nr_boosted = 0;
  6693. rt_rq->rq = rq;
  6694. #endif
  6695. }
  6696. #ifdef CONFIG_FAIR_GROUP_SCHED
  6697. static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
  6698. struct sched_entity *se, int cpu,
  6699. struct sched_entity *parent)
  6700. {
  6701. struct rq *rq = cpu_rq(cpu);
  6702. tg->cfs_rq[cpu] = cfs_rq;
  6703. init_cfs_rq(cfs_rq, rq);
  6704. cfs_rq->tg = tg;
  6705. tg->se[cpu] = se;
  6706. /* se could be NULL for root_task_group */
  6707. if (!se)
  6708. return;
  6709. if (!parent)
  6710. se->cfs_rq = &rq->cfs;
  6711. else
  6712. se->cfs_rq = parent->my_q;
  6713. se->my_q = cfs_rq;
  6714. update_load_set(&se->load, 0);
  6715. se->parent = parent;
  6716. }
  6717. #endif
  6718. #ifdef CONFIG_RT_GROUP_SCHED
  6719. static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
  6720. struct sched_rt_entity *rt_se, int cpu,
  6721. struct sched_rt_entity *parent)
  6722. {
  6723. struct rq *rq = cpu_rq(cpu);
  6724. tg->rt_rq[cpu] = rt_rq;
  6725. init_rt_rq(rt_rq, rq);
  6726. rt_rq->tg = tg;
  6727. rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
  6728. tg->rt_se[cpu] = rt_se;
  6729. if (!rt_se)
  6730. return;
  6731. if (!parent)
  6732. rt_se->rt_rq = &rq->rt;
  6733. else
  6734. rt_se->rt_rq = parent->my_q;
  6735. rt_se->my_q = rt_rq;
  6736. rt_se->parent = parent;
  6737. INIT_LIST_HEAD(&rt_se->run_list);
  6738. }
  6739. #endif
  6740. void __init sched_init(void)
  6741. {
  6742. int i, j;
  6743. unsigned long alloc_size = 0, ptr;
  6744. #ifdef CONFIG_FAIR_GROUP_SCHED
  6745. alloc_size += 2 * nr_cpu_ids * sizeof(void **);
  6746. #endif
  6747. #ifdef CONFIG_RT_GROUP_SCHED
  6748. alloc_size += 2 * nr_cpu_ids * sizeof(void **);
  6749. #endif
  6750. #ifdef CONFIG_CPUMASK_OFFSTACK
  6751. alloc_size += num_possible_cpus() * cpumask_size();
  6752. #endif
  6753. if (alloc_size) {
  6754. ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
  6755. #ifdef CONFIG_FAIR_GROUP_SCHED
  6756. root_task_group.se = (struct sched_entity **)ptr;
  6757. ptr += nr_cpu_ids * sizeof(void **);
  6758. root_task_group.cfs_rq = (struct cfs_rq **)ptr;
  6759. ptr += nr_cpu_ids * sizeof(void **);
  6760. #endif /* CONFIG_FAIR_GROUP_SCHED */
  6761. #ifdef CONFIG_RT_GROUP_SCHED
  6762. root_task_group.rt_se = (struct sched_rt_entity **)ptr;
  6763. ptr += nr_cpu_ids * sizeof(void **);
  6764. root_task_group.rt_rq = (struct rt_rq **)ptr;
  6765. ptr += nr_cpu_ids * sizeof(void **);
  6766. #endif /* CONFIG_RT_GROUP_SCHED */
  6767. #ifdef CONFIG_CPUMASK_OFFSTACK
  6768. for_each_possible_cpu(i) {
  6769. per_cpu(load_balance_tmpmask, i) = (void *)ptr;
  6770. ptr += cpumask_size();
  6771. }
  6772. #endif /* CONFIG_CPUMASK_OFFSTACK */
  6773. }
  6774. #ifdef CONFIG_SMP
  6775. init_defrootdomain();
  6776. #endif
  6777. init_rt_bandwidth(&def_rt_bandwidth,
  6778. global_rt_period(), global_rt_runtime());
  6779. #ifdef CONFIG_RT_GROUP_SCHED
  6780. init_rt_bandwidth(&root_task_group.rt_bandwidth,
  6781. global_rt_period(), global_rt_runtime());
  6782. #endif /* CONFIG_RT_GROUP_SCHED */
  6783. #ifdef CONFIG_CGROUP_SCHED
  6784. list_add(&root_task_group.list, &task_groups);
  6785. INIT_LIST_HEAD(&root_task_group.children);
  6786. autogroup_init(&init_task);
  6787. #endif /* CONFIG_CGROUP_SCHED */
  6788. for_each_possible_cpu(i) {
  6789. struct rq *rq;
  6790. rq = cpu_rq(i);
  6791. raw_spin_lock_init(&rq->lock);
  6792. rq->nr_running = 0;
  6793. rq->calc_load_active = 0;
  6794. rq->calc_load_update = jiffies + LOAD_FREQ;
  6795. init_cfs_rq(&rq->cfs, rq);
  6796. init_rt_rq(&rq->rt, rq);
  6797. #ifdef CONFIG_FAIR_GROUP_SCHED
  6798. root_task_group.shares = root_task_group_load;
  6799. INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
  6800. /*
  6801. * How much cpu bandwidth does root_task_group get?
  6802. *
  6803. * In case of task-groups formed thr' the cgroup filesystem, it
  6804. * gets 100% of the cpu resources in the system. This overall
  6805. * system cpu resource is divided among the tasks of
  6806. * root_task_group and its child task-groups in a fair manner,
  6807. * based on each entity's (task or task-group's) weight
  6808. * (se->load.weight).
  6809. *
  6810. * In other words, if root_task_group has 10 tasks of weight
  6811. * 1024) and two child groups A0 and A1 (of weight 1024 each),
  6812. * then A0's share of the cpu resource is:
  6813. *
  6814. * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
  6815. *
  6816. * We achieve this by letting root_task_group's tasks sit
  6817. * directly in rq->cfs (i.e root_task_group->se[] = NULL).
  6818. */
  6819. init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
  6820. #endif /* CONFIG_FAIR_GROUP_SCHED */
  6821. rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
  6822. #ifdef CONFIG_RT_GROUP_SCHED
  6823. INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
  6824. init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
  6825. #endif
  6826. for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
  6827. rq->cpu_load[j] = 0;
  6828. rq->last_load_update_tick = jiffies;
  6829. #ifdef CONFIG_SMP
  6830. rq->sd = NULL;
  6831. rq->rd = NULL;
  6832. rq->cpu_power = SCHED_LOAD_SCALE;
  6833. rq->post_schedule = 0;
  6834. rq->active_balance = 0;
  6835. rq->next_balance = jiffies;
  6836. rq->push_cpu = 0;
  6837. rq->cpu = i;
  6838. rq->online = 0;
  6839. rq->idle_stamp = 0;
  6840. rq->avg_idle = 2*sysctl_sched_migration_cost;
  6841. rq_attach_root(rq, &def_root_domain);
  6842. #ifdef CONFIG_NO_HZ
  6843. rq->nohz_balance_kick = 0;
  6844. init_sched_softirq_csd(&per_cpu(remote_sched_softirq_cb, i));
  6845. #endif
  6846. #endif
  6847. init_rq_hrtick(rq);
  6848. atomic_set(&rq->nr_iowait, 0);
  6849. }
  6850. set_load_weight(&init_task);
  6851. #ifdef CONFIG_PREEMPT_NOTIFIERS
  6852. INIT_HLIST_HEAD(&init_task.preempt_notifiers);
  6853. #endif
  6854. #ifdef CONFIG_SMP
  6855. open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
  6856. #endif
  6857. #ifdef CONFIG_RT_MUTEXES
  6858. plist_head_init_raw(&init_task.pi_waiters, &init_task.pi_lock);
  6859. #endif
  6860. /*
  6861. * The boot idle thread does lazy MMU switching as well:
  6862. */
  6863. atomic_inc(&init_mm.mm_count);
  6864. enter_lazy_tlb(&init_mm, current);
  6865. /*
  6866. * Make us the idle thread. Technically, schedule() should not be
  6867. * called from this thread, however somewhere below it might be,
  6868. * but because we are the idle thread, we just pick up running again
  6869. * when this runqueue becomes "idle".
  6870. */
  6871. init_idle(current, smp_processor_id());
  6872. calc_load_update = jiffies + LOAD_FREQ;
  6873. /*
  6874. * During early bootup we pretend to be a normal task:
  6875. */
  6876. current->sched_class = &fair_sched_class;
  6877. /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
  6878. zalloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT);
  6879. #ifdef CONFIG_SMP
  6880. #ifdef CONFIG_NO_HZ
  6881. zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
  6882. alloc_cpumask_var(&nohz.grp_idle_mask, GFP_NOWAIT);
  6883. atomic_set(&nohz.load_balancer, nr_cpu_ids);
  6884. atomic_set(&nohz.first_pick_cpu, nr_cpu_ids);
  6885. atomic_set(&nohz.second_pick_cpu, nr_cpu_ids);
  6886. #endif
  6887. /* May be allocated at isolcpus cmdline parse time */
  6888. if (cpu_isolated_map == NULL)
  6889. zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
  6890. #endif /* SMP */
  6891. scheduler_running = 1;
  6892. }
  6893. #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
  6894. static inline int preempt_count_equals(int preempt_offset)
  6895. {
  6896. int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
  6897. return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
  6898. }
  6899. void __might_sleep(const char *file, int line, int preempt_offset)
  6900. {
  6901. #ifdef in_atomic
  6902. static unsigned long prev_jiffy; /* ratelimiting */
  6903. if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) ||
  6904. system_state != SYSTEM_RUNNING || oops_in_progress)
  6905. return;
  6906. if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
  6907. return;
  6908. prev_jiffy = jiffies;
  6909. printk(KERN_ERR
  6910. "BUG: sleeping function called from invalid context at %s:%d\n",
  6911. file, line);
  6912. printk(KERN_ERR
  6913. "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
  6914. in_atomic(), irqs_disabled(),
  6915. current->pid, current->comm);
  6916. debug_show_held_locks(current);
  6917. if (irqs_disabled())
  6918. print_irqtrace_events(current);
  6919. dump_stack();
  6920. #endif
  6921. }
  6922. EXPORT_SYMBOL(__might_sleep);
  6923. #endif
  6924. #ifdef CONFIG_MAGIC_SYSRQ
  6925. static void normalize_task(struct rq *rq, struct task_struct *p)
  6926. {
  6927. int on_rq;
  6928. on_rq = p->se.on_rq;
  6929. if (on_rq)
  6930. deactivate_task(rq, p, 0);
  6931. __setscheduler(rq, p, SCHED_NORMAL, 0);
  6932. if (on_rq) {
  6933. activate_task(rq, p, 0);
  6934. resched_task(rq->curr);
  6935. }
  6936. }
  6937. void normalize_rt_tasks(void)
  6938. {
  6939. struct task_struct *g, *p;
  6940. unsigned long flags;
  6941. struct rq *rq;
  6942. read_lock_irqsave(&tasklist_lock, flags);
  6943. do_each_thread(g, p) {
  6944. /*
  6945. * Only normalize user tasks:
  6946. */
  6947. if (!p->mm)
  6948. continue;
  6949. p->se.exec_start = 0;
  6950. #ifdef CONFIG_SCHEDSTATS
  6951. p->se.statistics.wait_start = 0;
  6952. p->se.statistics.sleep_start = 0;
  6953. p->se.statistics.block_start = 0;
  6954. #endif
  6955. if (!rt_task(p)) {
  6956. /*
  6957. * Renice negative nice level userspace
  6958. * tasks back to 0:
  6959. */
  6960. if (TASK_NICE(p) < 0 && p->mm)
  6961. set_user_nice(p, 0);
  6962. continue;
  6963. }
  6964. raw_spin_lock(&p->pi_lock);
  6965. rq = __task_rq_lock(p);
  6966. normalize_task(rq, p);
  6967. __task_rq_unlock(rq);
  6968. raw_spin_unlock(&p->pi_lock);
  6969. } while_each_thread(g, p);
  6970. read_unlock_irqrestore(&tasklist_lock, flags);
  6971. }
  6972. #endif /* CONFIG_MAGIC_SYSRQ */
  6973. #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
  6974. /*
  6975. * These functions are only useful for the IA64 MCA handling, or kdb.
  6976. *
  6977. * They can only be called when the whole system has been
  6978. * stopped - every CPU needs to be quiescent, and no scheduling
  6979. * activity can take place. Using them for anything else would
  6980. * be a serious bug, and as a result, they aren't even visible
  6981. * under any other configuration.
  6982. */
  6983. /**
  6984. * curr_task - return the current task for a given cpu.
  6985. * @cpu: the processor in question.
  6986. *
  6987. * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
  6988. */
  6989. struct task_struct *curr_task(int cpu)
  6990. {
  6991. return cpu_curr(cpu);
  6992. }
  6993. #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
  6994. #ifdef CONFIG_IA64
  6995. /**
  6996. * set_curr_task - set the current task for a given cpu.
  6997. * @cpu: the processor in question.
  6998. * @p: the task pointer to set.
  6999. *
  7000. * Description: This function must only be used when non-maskable interrupts
  7001. * are serviced on a separate stack. It allows the architecture to switch the
  7002. * notion of the current task on a cpu in a non-blocking manner. This function
  7003. * must be called with all CPU's synchronized, and interrupts disabled, the
  7004. * and caller must save the original value of the current task (see
  7005. * curr_task() above) and restore that value before reenabling interrupts and
  7006. * re-starting the system.
  7007. *
  7008. * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
  7009. */
  7010. void set_curr_task(int cpu, struct task_struct *p)
  7011. {
  7012. cpu_curr(cpu) = p;
  7013. }
  7014. #endif
  7015. #ifdef CONFIG_FAIR_GROUP_SCHED
  7016. static void free_fair_sched_group(struct task_group *tg)
  7017. {
  7018. int i;
  7019. for_each_possible_cpu(i) {
  7020. if (tg->cfs_rq)
  7021. kfree(tg->cfs_rq[i]);
  7022. if (tg->se)
  7023. kfree(tg->se[i]);
  7024. }
  7025. kfree(tg->cfs_rq);
  7026. kfree(tg->se);
  7027. }
  7028. static
  7029. int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
  7030. {
  7031. struct cfs_rq *cfs_rq;
  7032. struct sched_entity *se;
  7033. struct rq *rq;
  7034. int i;
  7035. tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
  7036. if (!tg->cfs_rq)
  7037. goto err;
  7038. tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
  7039. if (!tg->se)
  7040. goto err;
  7041. tg->shares = NICE_0_LOAD;
  7042. for_each_possible_cpu(i) {
  7043. rq = cpu_rq(i);
  7044. cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
  7045. GFP_KERNEL, cpu_to_node(i));
  7046. if (!cfs_rq)
  7047. goto err;
  7048. se = kzalloc_node(sizeof(struct sched_entity),
  7049. GFP_KERNEL, cpu_to_node(i));
  7050. if (!se)
  7051. goto err_free_rq;
  7052. init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
  7053. }
  7054. return 1;
  7055. err_free_rq:
  7056. kfree(cfs_rq);
  7057. err:
  7058. return 0;
  7059. }
  7060. static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
  7061. {
  7062. struct rq *rq = cpu_rq(cpu);
  7063. unsigned long flags;
  7064. /*
  7065. * Only empty task groups can be destroyed; so we can speculatively
  7066. * check on_list without danger of it being re-added.
  7067. */
  7068. if (!tg->cfs_rq[cpu]->on_list)
  7069. return;
  7070. raw_spin_lock_irqsave(&rq->lock, flags);
  7071. list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
  7072. raw_spin_unlock_irqrestore(&rq->lock, flags);
  7073. }
  7074. #else /* !CONFG_FAIR_GROUP_SCHED */
  7075. static inline void free_fair_sched_group(struct task_group *tg)
  7076. {
  7077. }
  7078. static inline
  7079. int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
  7080. {
  7081. return 1;
  7082. }
  7083. static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
  7084. {
  7085. }
  7086. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7087. #ifdef CONFIG_RT_GROUP_SCHED
  7088. static void free_rt_sched_group(struct task_group *tg)
  7089. {
  7090. int i;
  7091. destroy_rt_bandwidth(&tg->rt_bandwidth);
  7092. for_each_possible_cpu(i) {
  7093. if (tg->rt_rq)
  7094. kfree(tg->rt_rq[i]);
  7095. if (tg->rt_se)
  7096. kfree(tg->rt_se[i]);
  7097. }
  7098. kfree(tg->rt_rq);
  7099. kfree(tg->rt_se);
  7100. }
  7101. static
  7102. int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
  7103. {
  7104. struct rt_rq *rt_rq;
  7105. struct sched_rt_entity *rt_se;
  7106. struct rq *rq;
  7107. int i;
  7108. tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
  7109. if (!tg->rt_rq)
  7110. goto err;
  7111. tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
  7112. if (!tg->rt_se)
  7113. goto err;
  7114. init_rt_bandwidth(&tg->rt_bandwidth,
  7115. ktime_to_ns(def_rt_bandwidth.rt_period), 0);
  7116. for_each_possible_cpu(i) {
  7117. rq = cpu_rq(i);
  7118. rt_rq = kzalloc_node(sizeof(struct rt_rq),
  7119. GFP_KERNEL, cpu_to_node(i));
  7120. if (!rt_rq)
  7121. goto err;
  7122. rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
  7123. GFP_KERNEL, cpu_to_node(i));
  7124. if (!rt_se)
  7125. goto err_free_rq;
  7126. init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
  7127. }
  7128. return 1;
  7129. err_free_rq:
  7130. kfree(rt_rq);
  7131. err:
  7132. return 0;
  7133. }
  7134. #else /* !CONFIG_RT_GROUP_SCHED */
  7135. static inline void free_rt_sched_group(struct task_group *tg)
  7136. {
  7137. }
  7138. static inline
  7139. int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
  7140. {
  7141. return 1;
  7142. }
  7143. #endif /* CONFIG_RT_GROUP_SCHED */
  7144. #ifdef CONFIG_CGROUP_SCHED
  7145. static void free_sched_group(struct task_group *tg)
  7146. {
  7147. free_fair_sched_group(tg);
  7148. free_rt_sched_group(tg);
  7149. autogroup_free(tg);
  7150. kfree(tg);
  7151. }
  7152. /* allocate runqueue etc for a new task group */
  7153. struct task_group *sched_create_group(struct task_group *parent)
  7154. {
  7155. struct task_group *tg;
  7156. unsigned long flags;
  7157. tg = kzalloc(sizeof(*tg), GFP_KERNEL);
  7158. if (!tg)
  7159. return ERR_PTR(-ENOMEM);
  7160. if (!alloc_fair_sched_group(tg, parent))
  7161. goto err;
  7162. if (!alloc_rt_sched_group(tg, parent))
  7163. goto err;
  7164. spin_lock_irqsave(&task_group_lock, flags);
  7165. list_add_rcu(&tg->list, &task_groups);
  7166. WARN_ON(!parent); /* root should already exist */
  7167. tg->parent = parent;
  7168. INIT_LIST_HEAD(&tg->children);
  7169. list_add_rcu(&tg->siblings, &parent->children);
  7170. spin_unlock_irqrestore(&task_group_lock, flags);
  7171. return tg;
  7172. err:
  7173. free_sched_group(tg);
  7174. return ERR_PTR(-ENOMEM);
  7175. }
  7176. /* rcu callback to free various structures associated with a task group */
  7177. static void free_sched_group_rcu(struct rcu_head *rhp)
  7178. {
  7179. /* now it should be safe to free those cfs_rqs */
  7180. free_sched_group(container_of(rhp, struct task_group, rcu));
  7181. }
  7182. /* Destroy runqueue etc associated with a task group */
  7183. void sched_destroy_group(struct task_group *tg)
  7184. {
  7185. unsigned long flags;
  7186. int i;
  7187. /* end participation in shares distribution */
  7188. for_each_possible_cpu(i)
  7189. unregister_fair_sched_group(tg, i);
  7190. spin_lock_irqsave(&task_group_lock, flags);
  7191. list_del_rcu(&tg->list);
  7192. list_del_rcu(&tg->siblings);
  7193. spin_unlock_irqrestore(&task_group_lock, flags);
  7194. /* wait for possible concurrent references to cfs_rqs complete */
  7195. call_rcu(&tg->rcu, free_sched_group_rcu);
  7196. }
  7197. /* change task's runqueue when it moves between groups.
  7198. * The caller of this function should have put the task in its new group
  7199. * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
  7200. * reflect its new group.
  7201. */
  7202. void sched_move_task(struct task_struct *tsk)
  7203. {
  7204. int on_rq, running;
  7205. unsigned long flags;
  7206. struct rq *rq;
  7207. rq = task_rq_lock(tsk, &flags);
  7208. running = task_current(rq, tsk);
  7209. on_rq = tsk->se.on_rq;
  7210. if (on_rq)
  7211. dequeue_task(rq, tsk, 0);
  7212. if (unlikely(running))
  7213. tsk->sched_class->put_prev_task(rq, tsk);
  7214. #ifdef CONFIG_FAIR_GROUP_SCHED
  7215. if (tsk->sched_class->task_move_group)
  7216. tsk->sched_class->task_move_group(tsk, on_rq);
  7217. else
  7218. #endif
  7219. set_task_rq(tsk, task_cpu(tsk));
  7220. if (unlikely(running))
  7221. tsk->sched_class->set_curr_task(rq);
  7222. if (on_rq)
  7223. enqueue_task(rq, tsk, 0);
  7224. task_rq_unlock(rq, &flags);
  7225. }
  7226. #endif /* CONFIG_CGROUP_SCHED */
  7227. #ifdef CONFIG_FAIR_GROUP_SCHED
  7228. static DEFINE_MUTEX(shares_mutex);
  7229. int sched_group_set_shares(struct task_group *tg, unsigned long shares)
  7230. {
  7231. int i;
  7232. unsigned long flags;
  7233. /*
  7234. * We can't change the weight of the root cgroup.
  7235. */
  7236. if (!tg->se[0])
  7237. return -EINVAL;
  7238. if (shares < MIN_SHARES)
  7239. shares = MIN_SHARES;
  7240. else if (shares > MAX_SHARES)
  7241. shares = MAX_SHARES;
  7242. mutex_lock(&shares_mutex);
  7243. if (tg->shares == shares)
  7244. goto done;
  7245. tg->shares = shares;
  7246. for_each_possible_cpu(i) {
  7247. struct rq *rq = cpu_rq(i);
  7248. struct sched_entity *se;
  7249. se = tg->se[i];
  7250. /* Propagate contribution to hierarchy */
  7251. raw_spin_lock_irqsave(&rq->lock, flags);
  7252. for_each_sched_entity(se)
  7253. update_cfs_shares(group_cfs_rq(se), 0);
  7254. raw_spin_unlock_irqrestore(&rq->lock, flags);
  7255. }
  7256. done:
  7257. mutex_unlock(&shares_mutex);
  7258. return 0;
  7259. }
  7260. unsigned long sched_group_shares(struct task_group *tg)
  7261. {
  7262. return tg->shares;
  7263. }
  7264. #endif
  7265. #ifdef CONFIG_RT_GROUP_SCHED
  7266. /*
  7267. * Ensure that the real time constraints are schedulable.
  7268. */
  7269. static DEFINE_MUTEX(rt_constraints_mutex);
  7270. static unsigned long to_ratio(u64 period, u64 runtime)
  7271. {
  7272. if (runtime == RUNTIME_INF)
  7273. return 1ULL << 20;
  7274. return div64_u64(runtime << 20, period);
  7275. }
  7276. /* Must be called with tasklist_lock held */
  7277. static inline int tg_has_rt_tasks(struct task_group *tg)
  7278. {
  7279. struct task_struct *g, *p;
  7280. do_each_thread(g, p) {
  7281. if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg)
  7282. return 1;
  7283. } while_each_thread(g, p);
  7284. return 0;
  7285. }
  7286. struct rt_schedulable_data {
  7287. struct task_group *tg;
  7288. u64 rt_period;
  7289. u64 rt_runtime;
  7290. };
  7291. static int tg_schedulable(struct task_group *tg, void *data)
  7292. {
  7293. struct rt_schedulable_data *d = data;
  7294. struct task_group *child;
  7295. unsigned long total, sum = 0;
  7296. u64 period, runtime;
  7297. period = ktime_to_ns(tg->rt_bandwidth.rt_period);
  7298. runtime = tg->rt_bandwidth.rt_runtime;
  7299. if (tg == d->tg) {
  7300. period = d->rt_period;
  7301. runtime = d->rt_runtime;
  7302. }
  7303. /*
  7304. * Cannot have more runtime than the period.
  7305. */
  7306. if (runtime > period && runtime != RUNTIME_INF)
  7307. return -EINVAL;
  7308. /*
  7309. * Ensure we don't starve existing RT tasks.
  7310. */
  7311. if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
  7312. return -EBUSY;
  7313. total = to_ratio(period, runtime);
  7314. /*
  7315. * Nobody can have more than the global setting allows.
  7316. */
  7317. if (total > to_ratio(global_rt_period(), global_rt_runtime()))
  7318. return -EINVAL;
  7319. /*
  7320. * The sum of our children's runtime should not exceed our own.
  7321. */
  7322. list_for_each_entry_rcu(child, &tg->children, siblings) {
  7323. period = ktime_to_ns(child->rt_bandwidth.rt_period);
  7324. runtime = child->rt_bandwidth.rt_runtime;
  7325. if (child == d->tg) {
  7326. period = d->rt_period;
  7327. runtime = d->rt_runtime;
  7328. }
  7329. sum += to_ratio(period, runtime);
  7330. }
  7331. if (sum > total)
  7332. return -EINVAL;
  7333. return 0;
  7334. }
  7335. static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
  7336. {
  7337. struct rt_schedulable_data data = {
  7338. .tg = tg,
  7339. .rt_period = period,
  7340. .rt_runtime = runtime,
  7341. };
  7342. return walk_tg_tree(tg_schedulable, tg_nop, &data);
  7343. }
  7344. static int tg_set_bandwidth(struct task_group *tg,
  7345. u64 rt_period, u64 rt_runtime)
  7346. {
  7347. int i, err = 0;
  7348. mutex_lock(&rt_constraints_mutex);
  7349. read_lock(&tasklist_lock);
  7350. err = __rt_schedulable(tg, rt_period, rt_runtime);
  7351. if (err)
  7352. goto unlock;
  7353. raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  7354. tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
  7355. tg->rt_bandwidth.rt_runtime = rt_runtime;
  7356. for_each_possible_cpu(i) {
  7357. struct rt_rq *rt_rq = tg->rt_rq[i];
  7358. raw_spin_lock(&rt_rq->rt_runtime_lock);
  7359. rt_rq->rt_runtime = rt_runtime;
  7360. raw_spin_unlock(&rt_rq->rt_runtime_lock);
  7361. }
  7362. raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  7363. unlock:
  7364. read_unlock(&tasklist_lock);
  7365. mutex_unlock(&rt_constraints_mutex);
  7366. return err;
  7367. }
  7368. int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
  7369. {
  7370. u64 rt_runtime, rt_period;
  7371. rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
  7372. rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
  7373. if (rt_runtime_us < 0)
  7374. rt_runtime = RUNTIME_INF;
  7375. return tg_set_bandwidth(tg, rt_period, rt_runtime);
  7376. }
  7377. long sched_group_rt_runtime(struct task_group *tg)
  7378. {
  7379. u64 rt_runtime_us;
  7380. if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
  7381. return -1;
  7382. rt_runtime_us = tg->rt_bandwidth.rt_runtime;
  7383. do_div(rt_runtime_us, NSEC_PER_USEC);
  7384. return rt_runtime_us;
  7385. }
  7386. int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
  7387. {
  7388. u64 rt_runtime, rt_period;
  7389. rt_period = (u64)rt_period_us * NSEC_PER_USEC;
  7390. rt_runtime = tg->rt_bandwidth.rt_runtime;
  7391. if (rt_period == 0)
  7392. return -EINVAL;
  7393. return tg_set_bandwidth(tg, rt_period, rt_runtime);
  7394. }
  7395. long sched_group_rt_period(struct task_group *tg)
  7396. {
  7397. u64 rt_period_us;
  7398. rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
  7399. do_div(rt_period_us, NSEC_PER_USEC);
  7400. return rt_period_us;
  7401. }
  7402. static int sched_rt_global_constraints(void)
  7403. {
  7404. u64 runtime, period;
  7405. int ret = 0;
  7406. if (sysctl_sched_rt_period <= 0)
  7407. return -EINVAL;
  7408. runtime = global_rt_runtime();
  7409. period = global_rt_period();
  7410. /*
  7411. * Sanity check on the sysctl variables.
  7412. */
  7413. if (runtime > period && runtime != RUNTIME_INF)
  7414. return -EINVAL;
  7415. mutex_lock(&rt_constraints_mutex);
  7416. read_lock(&tasklist_lock);
  7417. ret = __rt_schedulable(NULL, 0, 0);
  7418. read_unlock(&tasklist_lock);
  7419. mutex_unlock(&rt_constraints_mutex);
  7420. return ret;
  7421. }
  7422. int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
  7423. {
  7424. /* Don't accept realtime tasks when there is no way for them to run */
  7425. if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
  7426. return 0;
  7427. return 1;
  7428. }
  7429. #else /* !CONFIG_RT_GROUP_SCHED */
  7430. static int sched_rt_global_constraints(void)
  7431. {
  7432. unsigned long flags;
  7433. int i;
  7434. if (sysctl_sched_rt_period <= 0)
  7435. return -EINVAL;
  7436. /*
  7437. * There's always some RT tasks in the root group
  7438. * -- migration, kstopmachine etc..
  7439. */
  7440. if (sysctl_sched_rt_runtime == 0)
  7441. return -EBUSY;
  7442. raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
  7443. for_each_possible_cpu(i) {
  7444. struct rt_rq *rt_rq = &cpu_rq(i)->rt;
  7445. raw_spin_lock(&rt_rq->rt_runtime_lock);
  7446. rt_rq->rt_runtime = global_rt_runtime();
  7447. raw_spin_unlock(&rt_rq->rt_runtime_lock);
  7448. }
  7449. raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
  7450. return 0;
  7451. }
  7452. #endif /* CONFIG_RT_GROUP_SCHED */
  7453. int sched_rt_handler(struct ctl_table *table, int write,
  7454. void __user *buffer, size_t *lenp,
  7455. loff_t *ppos)
  7456. {
  7457. int ret;
  7458. int old_period, old_runtime;
  7459. static DEFINE_MUTEX(mutex);
  7460. mutex_lock(&mutex);
  7461. old_period = sysctl_sched_rt_period;
  7462. old_runtime = sysctl_sched_rt_runtime;
  7463. ret = proc_dointvec(table, write, buffer, lenp, ppos);
  7464. if (!ret && write) {
  7465. ret = sched_rt_global_constraints();
  7466. if (ret) {
  7467. sysctl_sched_rt_period = old_period;
  7468. sysctl_sched_rt_runtime = old_runtime;
  7469. } else {
  7470. def_rt_bandwidth.rt_runtime = global_rt_runtime();
  7471. def_rt_bandwidth.rt_period =
  7472. ns_to_ktime(global_rt_period());
  7473. }
  7474. }
  7475. mutex_unlock(&mutex);
  7476. return ret;
  7477. }
  7478. #ifdef CONFIG_CGROUP_SCHED
  7479. /* return corresponding task_group object of a cgroup */
  7480. static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
  7481. {
  7482. return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
  7483. struct task_group, css);
  7484. }
  7485. static struct cgroup_subsys_state *
  7486. cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
  7487. {
  7488. struct task_group *tg, *parent;
  7489. if (!cgrp->parent) {
  7490. /* This is early initialization for the top cgroup */
  7491. return &root_task_group.css;
  7492. }
  7493. parent = cgroup_tg(cgrp->parent);
  7494. tg = sched_create_group(parent);
  7495. if (IS_ERR(tg))
  7496. return ERR_PTR(-ENOMEM);
  7497. return &tg->css;
  7498. }
  7499. static void
  7500. cpu_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
  7501. {
  7502. struct task_group *tg = cgroup_tg(cgrp);
  7503. sched_destroy_group(tg);
  7504. }
  7505. static int
  7506. cpu_cgroup_can_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
  7507. {
  7508. #ifdef CONFIG_RT_GROUP_SCHED
  7509. if (!sched_rt_can_attach(cgroup_tg(cgrp), tsk))
  7510. return -EINVAL;
  7511. #else
  7512. /* We don't support RT-tasks being in separate groups */
  7513. if (tsk->sched_class != &fair_sched_class)
  7514. return -EINVAL;
  7515. #endif
  7516. return 0;
  7517. }
  7518. static int
  7519. cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
  7520. struct task_struct *tsk, bool threadgroup)
  7521. {
  7522. int retval = cpu_cgroup_can_attach_task(cgrp, tsk);
  7523. if (retval)
  7524. return retval;
  7525. if (threadgroup) {
  7526. struct task_struct *c;
  7527. rcu_read_lock();
  7528. list_for_each_entry_rcu(c, &tsk->thread_group, thread_group) {
  7529. retval = cpu_cgroup_can_attach_task(cgrp, c);
  7530. if (retval) {
  7531. rcu_read_unlock();
  7532. return retval;
  7533. }
  7534. }
  7535. rcu_read_unlock();
  7536. }
  7537. return 0;
  7538. }
  7539. static void
  7540. cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
  7541. struct cgroup *old_cont, struct task_struct *tsk,
  7542. bool threadgroup)
  7543. {
  7544. sched_move_task(tsk);
  7545. if (threadgroup) {
  7546. struct task_struct *c;
  7547. rcu_read_lock();
  7548. list_for_each_entry_rcu(c, &tsk->thread_group, thread_group) {
  7549. sched_move_task(c);
  7550. }
  7551. rcu_read_unlock();
  7552. }
  7553. }
  7554. #ifdef CONFIG_FAIR_GROUP_SCHED
  7555. static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
  7556. u64 shareval)
  7557. {
  7558. return sched_group_set_shares(cgroup_tg(cgrp), shareval);
  7559. }
  7560. static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
  7561. {
  7562. struct task_group *tg = cgroup_tg(cgrp);
  7563. return (u64) tg->shares;
  7564. }
  7565. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7566. #ifdef CONFIG_RT_GROUP_SCHED
  7567. static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
  7568. s64 val)
  7569. {
  7570. return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
  7571. }
  7572. static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
  7573. {
  7574. return sched_group_rt_runtime(cgroup_tg(cgrp));
  7575. }
  7576. static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
  7577. u64 rt_period_us)
  7578. {
  7579. return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
  7580. }
  7581. static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
  7582. {
  7583. return sched_group_rt_period(cgroup_tg(cgrp));
  7584. }
  7585. #endif /* CONFIG_RT_GROUP_SCHED */
  7586. static struct cftype cpu_files[] = {
  7587. #ifdef CONFIG_FAIR_GROUP_SCHED
  7588. {
  7589. .name = "shares",
  7590. .read_u64 = cpu_shares_read_u64,
  7591. .write_u64 = cpu_shares_write_u64,
  7592. },
  7593. #endif
  7594. #ifdef CONFIG_RT_GROUP_SCHED
  7595. {
  7596. .name = "rt_runtime_us",
  7597. .read_s64 = cpu_rt_runtime_read,
  7598. .write_s64 = cpu_rt_runtime_write,
  7599. },
  7600. {
  7601. .name = "rt_period_us",
  7602. .read_u64 = cpu_rt_period_read_uint,
  7603. .write_u64 = cpu_rt_period_write_uint,
  7604. },
  7605. #endif
  7606. };
  7607. static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
  7608. {
  7609. return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files));
  7610. }
  7611. struct cgroup_subsys cpu_cgroup_subsys = {
  7612. .name = "cpu",
  7613. .create = cpu_cgroup_create,
  7614. .destroy = cpu_cgroup_destroy,
  7615. .can_attach = cpu_cgroup_can_attach,
  7616. .attach = cpu_cgroup_attach,
  7617. .populate = cpu_cgroup_populate,
  7618. .subsys_id = cpu_cgroup_subsys_id,
  7619. .early_init = 1,
  7620. };
  7621. #endif /* CONFIG_CGROUP_SCHED */
  7622. #ifdef CONFIG_CGROUP_CPUACCT
  7623. /*
  7624. * CPU accounting code for task groups.
  7625. *
  7626. * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
  7627. * (balbir@in.ibm.com).
  7628. */
  7629. /* track cpu usage of a group of tasks and its child groups */
  7630. struct cpuacct {
  7631. struct cgroup_subsys_state css;
  7632. /* cpuusage holds pointer to a u64-type object on every cpu */
  7633. u64 __percpu *cpuusage;
  7634. struct percpu_counter cpustat[CPUACCT_STAT_NSTATS];
  7635. struct cpuacct *parent;
  7636. };
  7637. struct cgroup_subsys cpuacct_subsys;
  7638. /* return cpu accounting group corresponding to this container */
  7639. static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
  7640. {
  7641. return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
  7642. struct cpuacct, css);
  7643. }
  7644. /* return cpu accounting group to which this task belongs */
  7645. static inline struct cpuacct *task_ca(struct task_struct *tsk)
  7646. {
  7647. return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
  7648. struct cpuacct, css);
  7649. }
  7650. /* create a new cpu accounting group */
  7651. static struct cgroup_subsys_state *cpuacct_create(
  7652. struct cgroup_subsys *ss, struct cgroup *cgrp)
  7653. {
  7654. struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
  7655. int i;
  7656. if (!ca)
  7657. goto out;
  7658. ca->cpuusage = alloc_percpu(u64);
  7659. if (!ca->cpuusage)
  7660. goto out_free_ca;
  7661. for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
  7662. if (percpu_counter_init(&ca->cpustat[i], 0))
  7663. goto out_free_counters;
  7664. if (cgrp->parent)
  7665. ca->parent = cgroup_ca(cgrp->parent);
  7666. return &ca->css;
  7667. out_free_counters:
  7668. while (--i >= 0)
  7669. percpu_counter_destroy(&ca->cpustat[i]);
  7670. free_percpu(ca->cpuusage);
  7671. out_free_ca:
  7672. kfree(ca);
  7673. out:
  7674. return ERR_PTR(-ENOMEM);
  7675. }
  7676. /* destroy an existing cpu accounting group */
  7677. static void
  7678. cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
  7679. {
  7680. struct cpuacct *ca = cgroup_ca(cgrp);
  7681. int i;
  7682. for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
  7683. percpu_counter_destroy(&ca->cpustat[i]);
  7684. free_percpu(ca->cpuusage);
  7685. kfree(ca);
  7686. }
  7687. static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
  7688. {
  7689. u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
  7690. u64 data;
  7691. #ifndef CONFIG_64BIT
  7692. /*
  7693. * Take rq->lock to make 64-bit read safe on 32-bit platforms.
  7694. */
  7695. raw_spin_lock_irq(&cpu_rq(cpu)->lock);
  7696. data = *cpuusage;
  7697. raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
  7698. #else
  7699. data = *cpuusage;
  7700. #endif
  7701. return data;
  7702. }
  7703. static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
  7704. {
  7705. u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
  7706. #ifndef CONFIG_64BIT
  7707. /*
  7708. * Take rq->lock to make 64-bit write safe on 32-bit platforms.
  7709. */
  7710. raw_spin_lock_irq(&cpu_rq(cpu)->lock);
  7711. *cpuusage = val;
  7712. raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
  7713. #else
  7714. *cpuusage = val;
  7715. #endif
  7716. }
  7717. /* return total cpu usage (in nanoseconds) of a group */
  7718. static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
  7719. {
  7720. struct cpuacct *ca = cgroup_ca(cgrp);
  7721. u64 totalcpuusage = 0;
  7722. int i;
  7723. for_each_present_cpu(i)
  7724. totalcpuusage += cpuacct_cpuusage_read(ca, i);
  7725. return totalcpuusage;
  7726. }
  7727. static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
  7728. u64 reset)
  7729. {
  7730. struct cpuacct *ca = cgroup_ca(cgrp);
  7731. int err = 0;
  7732. int i;
  7733. if (reset) {
  7734. err = -EINVAL;
  7735. goto out;
  7736. }
  7737. for_each_present_cpu(i)
  7738. cpuacct_cpuusage_write(ca, i, 0);
  7739. out:
  7740. return err;
  7741. }
  7742. static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
  7743. struct seq_file *m)
  7744. {
  7745. struct cpuacct *ca = cgroup_ca(cgroup);
  7746. u64 percpu;
  7747. int i;
  7748. for_each_present_cpu(i) {
  7749. percpu = cpuacct_cpuusage_read(ca, i);
  7750. seq_printf(m, "%llu ", (unsigned long long) percpu);
  7751. }
  7752. seq_printf(m, "\n");
  7753. return 0;
  7754. }
  7755. static const char *cpuacct_stat_desc[] = {
  7756. [CPUACCT_STAT_USER] = "user",
  7757. [CPUACCT_STAT_SYSTEM] = "system",
  7758. };
  7759. static int cpuacct_stats_show(struct cgroup *cgrp, struct cftype *cft,
  7760. struct cgroup_map_cb *cb)
  7761. {
  7762. struct cpuacct *ca = cgroup_ca(cgrp);
  7763. int i;
  7764. for (i = 0; i < CPUACCT_STAT_NSTATS; i++) {
  7765. s64 val = percpu_counter_read(&ca->cpustat[i]);
  7766. val = cputime64_to_clock_t(val);
  7767. cb->fill(cb, cpuacct_stat_desc[i], val);
  7768. }
  7769. return 0;
  7770. }
  7771. static struct cftype files[] = {
  7772. {
  7773. .name = "usage",
  7774. .read_u64 = cpuusage_read,
  7775. .write_u64 = cpuusage_write,
  7776. },
  7777. {
  7778. .name = "usage_percpu",
  7779. .read_seq_string = cpuacct_percpu_seq_read,
  7780. },
  7781. {
  7782. .name = "stat",
  7783. .read_map = cpuacct_stats_show,
  7784. },
  7785. };
  7786. static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
  7787. {
  7788. return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
  7789. }
  7790. /*
  7791. * charge this task's execution time to its accounting group.
  7792. *
  7793. * called with rq->lock held.
  7794. */
  7795. static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
  7796. {
  7797. struct cpuacct *ca;
  7798. int cpu;
  7799. if (unlikely(!cpuacct_subsys.active))
  7800. return;
  7801. cpu = task_cpu(tsk);
  7802. rcu_read_lock();
  7803. ca = task_ca(tsk);
  7804. for (; ca; ca = ca->parent) {
  7805. u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
  7806. *cpuusage += cputime;
  7807. }
  7808. rcu_read_unlock();
  7809. }
  7810. /*
  7811. * When CONFIG_VIRT_CPU_ACCOUNTING is enabled one jiffy can be very large
  7812. * in cputime_t units. As a result, cpuacct_update_stats calls
  7813. * percpu_counter_add with values large enough to always overflow the
  7814. * per cpu batch limit causing bad SMP scalability.
  7815. *
  7816. * To fix this we scale percpu_counter_batch by cputime_one_jiffy so we
  7817. * batch the same amount of time with CONFIG_VIRT_CPU_ACCOUNTING disabled
  7818. * and enabled. We cap it at INT_MAX which is the largest allowed batch value.
  7819. */
  7820. #ifdef CONFIG_SMP
  7821. #define CPUACCT_BATCH \
  7822. min_t(long, percpu_counter_batch * cputime_one_jiffy, INT_MAX)
  7823. #else
  7824. #define CPUACCT_BATCH 0
  7825. #endif
  7826. /*
  7827. * Charge the system/user time to the task's accounting group.
  7828. */
  7829. static void cpuacct_update_stats(struct task_struct *tsk,
  7830. enum cpuacct_stat_index idx, cputime_t val)
  7831. {
  7832. struct cpuacct *ca;
  7833. int batch = CPUACCT_BATCH;
  7834. if (unlikely(!cpuacct_subsys.active))
  7835. return;
  7836. rcu_read_lock();
  7837. ca = task_ca(tsk);
  7838. do {
  7839. __percpu_counter_add(&ca->cpustat[idx], val, batch);
  7840. ca = ca->parent;
  7841. } while (ca);
  7842. rcu_read_unlock();
  7843. }
  7844. struct cgroup_subsys cpuacct_subsys = {
  7845. .name = "cpuacct",
  7846. .create = cpuacct_create,
  7847. .destroy = cpuacct_destroy,
  7848. .populate = cpuacct_populate,
  7849. .subsys_id = cpuacct_subsys_id,
  7850. };
  7851. #endif /* CONFIG_CGROUP_CPUACCT */