sched.c 232 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586
  1. /*
  2. * kernel/sched.c
  3. *
  4. * Kernel scheduler and related syscalls
  5. *
  6. * Copyright (C) 1991-2002 Linus Torvalds
  7. *
  8. * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and
  9. * make semaphores SMP safe
  10. * 1998-11-19 Implemented schedule_timeout() and related stuff
  11. * by Andrea Arcangeli
  12. * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:
  13. * hybrid priority-list and round-robin design with
  14. * an array-switch method of distributing timeslices
  15. * and per-CPU runqueues. Cleanups and useful suggestions
  16. * by Davide Libenzi, preemptible kernel bits by Robert Love.
  17. * 2003-09-03 Interactivity tuning by Con Kolivas.
  18. * 2004-04-02 Scheduler domains code by Nick Piggin
  19. * 2007-04-15 Work begun on replacing all interactivity tuning with a
  20. * fair scheduling design by Con Kolivas.
  21. * 2007-05-05 Load balancing (smp-nice) and other improvements
  22. * by Peter Williams
  23. * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith
  24. * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri
  25. * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins,
  26. * Thomas Gleixner, Mike Kravetz
  27. */
  28. #include <linux/mm.h>
  29. #include <linux/module.h>
  30. #include <linux/nmi.h>
  31. #include <linux/init.h>
  32. #include <linux/uaccess.h>
  33. #include <linux/highmem.h>
  34. #include <linux/smp_lock.h>
  35. #include <asm/mmu_context.h>
  36. #include <linux/interrupt.h>
  37. #include <linux/capability.h>
  38. #include <linux/completion.h>
  39. #include <linux/kernel_stat.h>
  40. #include <linux/debug_locks.h>
  41. #include <linux/security.h>
  42. #include <linux/notifier.h>
  43. #include <linux/profile.h>
  44. #include <linux/freezer.h>
  45. #include <linux/vmalloc.h>
  46. #include <linux/blkdev.h>
  47. #include <linux/delay.h>
  48. #include <linux/pid_namespace.h>
  49. #include <linux/smp.h>
  50. #include <linux/threads.h>
  51. #include <linux/timer.h>
  52. #include <linux/rcupdate.h>
  53. #include <linux/cpu.h>
  54. #include <linux/cpuset.h>
  55. #include <linux/percpu.h>
  56. #include <linux/kthread.h>
  57. #include <linux/proc_fs.h>
  58. #include <linux/seq_file.h>
  59. #include <linux/sysctl.h>
  60. #include <linux/syscalls.h>
  61. #include <linux/times.h>
  62. #include <linux/tsacct_kern.h>
  63. #include <linux/kprobes.h>
  64. #include <linux/delayacct.h>
  65. #include <linux/reciprocal_div.h>
  66. #include <linux/unistd.h>
  67. #include <linux/pagemap.h>
  68. #include <linux/hrtimer.h>
  69. #include <linux/tick.h>
  70. #include <linux/bootmem.h>
  71. #include <linux/debugfs.h>
  72. #include <linux/ctype.h>
  73. #include <linux/ftrace.h>
  74. #include <trace/sched.h>
  75. #include <asm/tlb.h>
  76. #include <asm/irq_regs.h>
  77. #include "sched_cpupri.h"
  78. /*
  79. * Convert user-nice values [ -20 ... 0 ... 19 ]
  80. * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
  81. * and back.
  82. */
  83. #define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20)
  84. #define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20)
  85. #define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio)
  86. /*
  87. * 'User priority' is the nice value converted to something we
  88. * can work with better when scaling various scheduler parameters,
  89. * it's a [ 0 ... 39 ] range.
  90. */
  91. #define USER_PRIO(p) ((p)-MAX_RT_PRIO)
  92. #define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)
  93. #define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
  94. /*
  95. * Helpers for converting nanosecond timing to jiffy resolution
  96. */
  97. #define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
  98. #define NICE_0_LOAD SCHED_LOAD_SCALE
  99. #define NICE_0_SHIFT SCHED_LOAD_SHIFT
  100. /*
  101. * These are the 'tuning knobs' of the scheduler:
  102. *
  103. * default timeslice is 100 msecs (used only for SCHED_RR tasks).
  104. * Timeslices get refilled after they expire.
  105. */
  106. #define DEF_TIMESLICE (100 * HZ / 1000)
  107. /*
  108. * single value that denotes runtime == period, ie unlimited time.
  109. */
  110. #define RUNTIME_INF ((u64)~0ULL)
  111. DEFINE_TRACE(sched_wait_task);
  112. DEFINE_TRACE(sched_wakeup);
  113. DEFINE_TRACE(sched_wakeup_new);
  114. DEFINE_TRACE(sched_switch);
  115. DEFINE_TRACE(sched_migrate_task);
  116. #ifdef CONFIG_SMP
  117. /*
  118. * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
  119. * Since cpu_power is a 'constant', we can use a reciprocal divide.
  120. */
  121. static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
  122. {
  123. return reciprocal_divide(load, sg->reciprocal_cpu_power);
  124. }
  125. /*
  126. * Each time a sched group cpu_power is changed,
  127. * we must compute its reciprocal value
  128. */
  129. static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
  130. {
  131. sg->__cpu_power += val;
  132. sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
  133. }
  134. #endif
  135. static inline int rt_policy(int policy)
  136. {
  137. if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
  138. return 1;
  139. return 0;
  140. }
  141. static inline int task_has_rt_policy(struct task_struct *p)
  142. {
  143. return rt_policy(p->policy);
  144. }
  145. /*
  146. * This is the priority-queue data structure of the RT scheduling class:
  147. */
  148. struct rt_prio_array {
  149. DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
  150. struct list_head queue[MAX_RT_PRIO];
  151. };
  152. struct rt_bandwidth {
  153. /* nests inside the rq lock: */
  154. spinlock_t rt_runtime_lock;
  155. ktime_t rt_period;
  156. u64 rt_runtime;
  157. struct hrtimer rt_period_timer;
  158. };
  159. static struct rt_bandwidth def_rt_bandwidth;
  160. static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
  161. static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
  162. {
  163. struct rt_bandwidth *rt_b =
  164. container_of(timer, struct rt_bandwidth, rt_period_timer);
  165. ktime_t now;
  166. int overrun;
  167. int idle = 0;
  168. for (;;) {
  169. now = hrtimer_cb_get_time(timer);
  170. overrun = hrtimer_forward(timer, now, rt_b->rt_period);
  171. if (!overrun)
  172. break;
  173. idle = do_sched_rt_period_timer(rt_b, overrun);
  174. }
  175. return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
  176. }
  177. static
  178. void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
  179. {
  180. rt_b->rt_period = ns_to_ktime(period);
  181. rt_b->rt_runtime = runtime;
  182. spin_lock_init(&rt_b->rt_runtime_lock);
  183. hrtimer_init(&rt_b->rt_period_timer,
  184. CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  185. rt_b->rt_period_timer.function = sched_rt_period_timer;
  186. }
  187. static inline int rt_bandwidth_enabled(void)
  188. {
  189. return sysctl_sched_rt_runtime >= 0;
  190. }
  191. static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
  192. {
  193. ktime_t now;
  194. if (rt_bandwidth_enabled() && rt_b->rt_runtime == RUNTIME_INF)
  195. return;
  196. if (hrtimer_active(&rt_b->rt_period_timer))
  197. return;
  198. spin_lock(&rt_b->rt_runtime_lock);
  199. for (;;) {
  200. if (hrtimer_active(&rt_b->rt_period_timer))
  201. break;
  202. now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
  203. hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
  204. hrtimer_start_expires(&rt_b->rt_period_timer,
  205. HRTIMER_MODE_ABS);
  206. }
  207. spin_unlock(&rt_b->rt_runtime_lock);
  208. }
  209. #ifdef CONFIG_RT_GROUP_SCHED
  210. static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
  211. {
  212. hrtimer_cancel(&rt_b->rt_period_timer);
  213. }
  214. #endif
  215. /*
  216. * sched_domains_mutex serializes calls to arch_init_sched_domains,
  217. * detach_destroy_domains and partition_sched_domains.
  218. */
  219. static DEFINE_MUTEX(sched_domains_mutex);
  220. #ifdef CONFIG_GROUP_SCHED
  221. #include <linux/cgroup.h>
  222. struct cfs_rq;
  223. static LIST_HEAD(task_groups);
  224. /* task group related information */
  225. struct task_group {
  226. #ifdef CONFIG_CGROUP_SCHED
  227. struct cgroup_subsys_state css;
  228. #endif
  229. #ifdef CONFIG_USER_SCHED
  230. uid_t uid;
  231. #endif
  232. #ifdef CONFIG_FAIR_GROUP_SCHED
  233. /* schedulable entities of this group on each cpu */
  234. struct sched_entity **se;
  235. /* runqueue "owned" by this group on each cpu */
  236. struct cfs_rq **cfs_rq;
  237. unsigned long shares;
  238. #endif
  239. #ifdef CONFIG_RT_GROUP_SCHED
  240. struct sched_rt_entity **rt_se;
  241. struct rt_rq **rt_rq;
  242. struct rt_bandwidth rt_bandwidth;
  243. #endif
  244. struct rcu_head rcu;
  245. struct list_head list;
  246. struct task_group *parent;
  247. struct list_head siblings;
  248. struct list_head children;
  249. };
  250. #ifdef CONFIG_USER_SCHED
  251. /* Helper function to pass uid information to create_sched_user() */
  252. void set_tg_uid(struct user_struct *user)
  253. {
  254. user->tg->uid = user->uid;
  255. }
  256. /*
  257. * Root task group.
  258. * Every UID task group (including init_task_group aka UID-0) will
  259. * be a child to this group.
  260. */
  261. struct task_group root_task_group;
  262. #ifdef CONFIG_FAIR_GROUP_SCHED
  263. /* Default task group's sched entity on each cpu */
  264. static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
  265. /* Default task group's cfs_rq on each cpu */
  266. static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
  267. #endif /* CONFIG_FAIR_GROUP_SCHED */
  268. #ifdef CONFIG_RT_GROUP_SCHED
  269. static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
  270. static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
  271. #endif /* CONFIG_RT_GROUP_SCHED */
  272. #else /* !CONFIG_USER_SCHED */
  273. #define root_task_group init_task_group
  274. #endif /* CONFIG_USER_SCHED */
  275. /* task_group_lock serializes add/remove of task groups and also changes to
  276. * a task group's cpu shares.
  277. */
  278. static DEFINE_SPINLOCK(task_group_lock);
  279. #ifdef CONFIG_FAIR_GROUP_SCHED
  280. #ifdef CONFIG_USER_SCHED
  281. # define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD)
  282. #else /* !CONFIG_USER_SCHED */
  283. # define INIT_TASK_GROUP_LOAD NICE_0_LOAD
  284. #endif /* CONFIG_USER_SCHED */
  285. /*
  286. * A weight of 0 or 1 can cause arithmetics problems.
  287. * A weight of a cfs_rq is the sum of weights of which entities
  288. * are queued on this cfs_rq, so a weight of a entity should not be
  289. * too large, so as the shares value of a task group.
  290. * (The default weight is 1024 - so there's no practical
  291. * limitation from this.)
  292. */
  293. #define MIN_SHARES 2
  294. #define MAX_SHARES (1UL << 18)
  295. static int init_task_group_load = INIT_TASK_GROUP_LOAD;
  296. #endif
  297. /* Default task group.
  298. * Every task in system belong to this group at bootup.
  299. */
  300. struct task_group init_task_group;
  301. /* return group to which a task belongs */
  302. static inline struct task_group *task_group(struct task_struct *p)
  303. {
  304. struct task_group *tg;
  305. #ifdef CONFIG_USER_SCHED
  306. rcu_read_lock();
  307. tg = __task_cred(p)->user->tg;
  308. rcu_read_unlock();
  309. #elif defined(CONFIG_CGROUP_SCHED)
  310. tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
  311. struct task_group, css);
  312. #else
  313. tg = &init_task_group;
  314. #endif
  315. return tg;
  316. }
  317. /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
  318. static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
  319. {
  320. #ifdef CONFIG_FAIR_GROUP_SCHED
  321. p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
  322. p->se.parent = task_group(p)->se[cpu];
  323. #endif
  324. #ifdef CONFIG_RT_GROUP_SCHED
  325. p->rt.rt_rq = task_group(p)->rt_rq[cpu];
  326. p->rt.parent = task_group(p)->rt_se[cpu];
  327. #endif
  328. }
  329. #else
  330. static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
  331. static inline struct task_group *task_group(struct task_struct *p)
  332. {
  333. return NULL;
  334. }
  335. #endif /* CONFIG_GROUP_SCHED */
  336. /* CFS-related fields in a runqueue */
  337. struct cfs_rq {
  338. struct load_weight load;
  339. unsigned long nr_running;
  340. u64 exec_clock;
  341. u64 min_vruntime;
  342. struct rb_root tasks_timeline;
  343. struct rb_node *rb_leftmost;
  344. struct list_head tasks;
  345. struct list_head *balance_iterator;
  346. /*
  347. * 'curr' points to currently running entity on this cfs_rq.
  348. * It is set to NULL otherwise (i.e when none are currently running).
  349. */
  350. struct sched_entity *curr, *next, *last;
  351. unsigned int nr_spread_over;
  352. #ifdef CONFIG_FAIR_GROUP_SCHED
  353. struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */
  354. /*
  355. * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
  356. * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
  357. * (like users, containers etc.)
  358. *
  359. * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
  360. * list is used during load balance.
  361. */
  362. struct list_head leaf_cfs_rq_list;
  363. struct task_group *tg; /* group that "owns" this runqueue */
  364. #ifdef CONFIG_SMP
  365. /*
  366. * the part of load.weight contributed by tasks
  367. */
  368. unsigned long task_weight;
  369. /*
  370. * h_load = weight * f(tg)
  371. *
  372. * Where f(tg) is the recursive weight fraction assigned to
  373. * this group.
  374. */
  375. unsigned long h_load;
  376. /*
  377. * this cpu's part of tg->shares
  378. */
  379. unsigned long shares;
  380. /*
  381. * load.weight at the time we set shares
  382. */
  383. unsigned long rq_weight;
  384. #endif
  385. #endif
  386. };
  387. /* Real-Time classes' related field in a runqueue: */
  388. struct rt_rq {
  389. struct rt_prio_array active;
  390. unsigned long rt_nr_running;
  391. #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
  392. int highest_prio; /* highest queued rt task prio */
  393. #endif
  394. #ifdef CONFIG_SMP
  395. unsigned long rt_nr_migratory;
  396. int overloaded;
  397. #endif
  398. int rt_throttled;
  399. u64 rt_time;
  400. u64 rt_runtime;
  401. /* Nests inside the rq lock: */
  402. spinlock_t rt_runtime_lock;
  403. #ifdef CONFIG_RT_GROUP_SCHED
  404. unsigned long rt_nr_boosted;
  405. struct rq *rq;
  406. struct list_head leaf_rt_rq_list;
  407. struct task_group *tg;
  408. struct sched_rt_entity *rt_se;
  409. #endif
  410. };
  411. #ifdef CONFIG_SMP
  412. /*
  413. * We add the notion of a root-domain which will be used to define per-domain
  414. * variables. Each exclusive cpuset essentially defines an island domain by
  415. * fully partitioning the member cpus from any other cpuset. Whenever a new
  416. * exclusive cpuset is created, we also create and attach a new root-domain
  417. * object.
  418. *
  419. */
  420. struct root_domain {
  421. atomic_t refcount;
  422. cpumask_var_t span;
  423. cpumask_var_t online;
  424. /*
  425. * The "RT overload" flag: it gets set if a CPU has more than
  426. * one runnable RT task.
  427. */
  428. cpumask_var_t rto_mask;
  429. atomic_t rto_count;
  430. #ifdef CONFIG_SMP
  431. struct cpupri cpupri;
  432. #endif
  433. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  434. /*
  435. * Preferred wake up cpu nominated by sched_mc balance that will be
  436. * used when most cpus are idle in the system indicating overall very
  437. * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
  438. */
  439. unsigned int sched_mc_preferred_wakeup_cpu;
  440. #endif
  441. };
  442. /*
  443. * By default the system creates a single root-domain with all cpus as
  444. * members (mimicking the global state we have today).
  445. */
  446. static struct root_domain def_root_domain;
  447. #endif
  448. /*
  449. * This is the main, per-CPU runqueue data structure.
  450. *
  451. * Locking rule: those places that want to lock multiple runqueues
  452. * (such as the load balancing or the thread migration code), lock
  453. * acquire operations must be ordered by ascending &runqueue.
  454. */
  455. struct rq {
  456. /* runqueue lock: */
  457. spinlock_t lock;
  458. /*
  459. * nr_running and cpu_load should be in the same cacheline because
  460. * remote CPUs use both these fields when doing load calculation.
  461. */
  462. unsigned long nr_running;
  463. #define CPU_LOAD_IDX_MAX 5
  464. unsigned long cpu_load[CPU_LOAD_IDX_MAX];
  465. unsigned char idle_at_tick;
  466. #ifdef CONFIG_NO_HZ
  467. unsigned long last_tick_seen;
  468. unsigned char in_nohz_recently;
  469. #endif
  470. /* capture load from *all* tasks on this cpu: */
  471. struct load_weight load;
  472. unsigned long nr_load_updates;
  473. u64 nr_switches;
  474. struct cfs_rq cfs;
  475. struct rt_rq rt;
  476. #ifdef CONFIG_FAIR_GROUP_SCHED
  477. /* list of leaf cfs_rq on this cpu: */
  478. struct list_head leaf_cfs_rq_list;
  479. #endif
  480. #ifdef CONFIG_RT_GROUP_SCHED
  481. struct list_head leaf_rt_rq_list;
  482. #endif
  483. /*
  484. * This is part of a global counter where only the total sum
  485. * over all CPUs matters. A task can increase this counter on
  486. * one CPU and if it got migrated afterwards it may decrease
  487. * it on another CPU. Always updated under the runqueue lock:
  488. */
  489. unsigned long nr_uninterruptible;
  490. struct task_struct *curr, *idle;
  491. unsigned long next_balance;
  492. struct mm_struct *prev_mm;
  493. u64 clock;
  494. atomic_t nr_iowait;
  495. #ifdef CONFIG_SMP
  496. struct root_domain *rd;
  497. struct sched_domain *sd;
  498. /* For active balancing */
  499. int active_balance;
  500. int push_cpu;
  501. /* cpu of this runqueue: */
  502. int cpu;
  503. int online;
  504. unsigned long avg_load_per_task;
  505. struct task_struct *migration_thread;
  506. struct list_head migration_queue;
  507. #endif
  508. #ifdef CONFIG_SCHED_HRTICK
  509. #ifdef CONFIG_SMP
  510. int hrtick_csd_pending;
  511. struct call_single_data hrtick_csd;
  512. #endif
  513. struct hrtimer hrtick_timer;
  514. #endif
  515. #ifdef CONFIG_SCHEDSTATS
  516. /* latency stats */
  517. struct sched_info rq_sched_info;
  518. unsigned long long rq_cpu_time;
  519. /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
  520. /* sys_sched_yield() stats */
  521. unsigned int yld_exp_empty;
  522. unsigned int yld_act_empty;
  523. unsigned int yld_both_empty;
  524. unsigned int yld_count;
  525. /* schedule() stats */
  526. unsigned int sched_switch;
  527. unsigned int sched_count;
  528. unsigned int sched_goidle;
  529. /* try_to_wake_up() stats */
  530. unsigned int ttwu_count;
  531. unsigned int ttwu_local;
  532. /* BKL stats */
  533. unsigned int bkl_count;
  534. #endif
  535. };
  536. static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
  537. static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync)
  538. {
  539. rq->curr->sched_class->check_preempt_curr(rq, p, sync);
  540. }
  541. static inline int cpu_of(struct rq *rq)
  542. {
  543. #ifdef CONFIG_SMP
  544. return rq->cpu;
  545. #else
  546. return 0;
  547. #endif
  548. }
  549. /*
  550. * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
  551. * See detach_destroy_domains: synchronize_sched for details.
  552. *
  553. * The domain tree of any CPU may only be accessed from within
  554. * preempt-disabled sections.
  555. */
  556. #define for_each_domain(cpu, __sd) \
  557. for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
  558. #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
  559. #define this_rq() (&__get_cpu_var(runqueues))
  560. #define task_rq(p) cpu_rq(task_cpu(p))
  561. #define cpu_curr(cpu) (cpu_rq(cpu)->curr)
  562. static inline void update_rq_clock(struct rq *rq)
  563. {
  564. rq->clock = sched_clock_cpu(cpu_of(rq));
  565. }
  566. /*
  567. * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
  568. */
  569. #ifdef CONFIG_SCHED_DEBUG
  570. # define const_debug __read_mostly
  571. #else
  572. # define const_debug static const
  573. #endif
  574. /**
  575. * runqueue_is_locked
  576. *
  577. * Returns true if the current cpu runqueue is locked.
  578. * This interface allows printk to be called with the runqueue lock
  579. * held and know whether or not it is OK to wake up the klogd.
  580. */
  581. int runqueue_is_locked(void)
  582. {
  583. int cpu = get_cpu();
  584. struct rq *rq = cpu_rq(cpu);
  585. int ret;
  586. ret = spin_is_locked(&rq->lock);
  587. put_cpu();
  588. return ret;
  589. }
  590. /*
  591. * Debugging: various feature bits
  592. */
  593. #define SCHED_FEAT(name, enabled) \
  594. __SCHED_FEAT_##name ,
  595. enum {
  596. #include "sched_features.h"
  597. };
  598. #undef SCHED_FEAT
  599. #define SCHED_FEAT(name, enabled) \
  600. (1UL << __SCHED_FEAT_##name) * enabled |
  601. const_debug unsigned int sysctl_sched_features =
  602. #include "sched_features.h"
  603. 0;
  604. #undef SCHED_FEAT
  605. #ifdef CONFIG_SCHED_DEBUG
  606. #define SCHED_FEAT(name, enabled) \
  607. #name ,
  608. static __read_mostly char *sched_feat_names[] = {
  609. #include "sched_features.h"
  610. NULL
  611. };
  612. #undef SCHED_FEAT
  613. static int sched_feat_show(struct seq_file *m, void *v)
  614. {
  615. int i;
  616. for (i = 0; sched_feat_names[i]; i++) {
  617. if (!(sysctl_sched_features & (1UL << i)))
  618. seq_puts(m, "NO_");
  619. seq_printf(m, "%s ", sched_feat_names[i]);
  620. }
  621. seq_puts(m, "\n");
  622. return 0;
  623. }
  624. static ssize_t
  625. sched_feat_write(struct file *filp, const char __user *ubuf,
  626. size_t cnt, loff_t *ppos)
  627. {
  628. char buf[64];
  629. char *cmp = buf;
  630. int neg = 0;
  631. int i;
  632. if (cnt > 63)
  633. cnt = 63;
  634. if (copy_from_user(&buf, ubuf, cnt))
  635. return -EFAULT;
  636. buf[cnt] = 0;
  637. if (strncmp(buf, "NO_", 3) == 0) {
  638. neg = 1;
  639. cmp += 3;
  640. }
  641. for (i = 0; sched_feat_names[i]; i++) {
  642. int len = strlen(sched_feat_names[i]);
  643. if (strncmp(cmp, sched_feat_names[i], len) == 0) {
  644. if (neg)
  645. sysctl_sched_features &= ~(1UL << i);
  646. else
  647. sysctl_sched_features |= (1UL << i);
  648. break;
  649. }
  650. }
  651. if (!sched_feat_names[i])
  652. return -EINVAL;
  653. filp->f_pos += cnt;
  654. return cnt;
  655. }
  656. static int sched_feat_open(struct inode *inode, struct file *filp)
  657. {
  658. return single_open(filp, sched_feat_show, NULL);
  659. }
  660. static struct file_operations sched_feat_fops = {
  661. .open = sched_feat_open,
  662. .write = sched_feat_write,
  663. .read = seq_read,
  664. .llseek = seq_lseek,
  665. .release = single_release,
  666. };
  667. static __init int sched_init_debug(void)
  668. {
  669. debugfs_create_file("sched_features", 0644, NULL, NULL,
  670. &sched_feat_fops);
  671. return 0;
  672. }
  673. late_initcall(sched_init_debug);
  674. #endif
  675. #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
  676. /*
  677. * Number of tasks to iterate in a single balance run.
  678. * Limited because this is done with IRQs disabled.
  679. */
  680. const_debug unsigned int sysctl_sched_nr_migrate = 32;
  681. /*
  682. * ratelimit for updating the group shares.
  683. * default: 0.25ms
  684. */
  685. unsigned int sysctl_sched_shares_ratelimit = 250000;
  686. /*
  687. * Inject some fuzzyness into changing the per-cpu group shares
  688. * this avoids remote rq-locks at the expense of fairness.
  689. * default: 4
  690. */
  691. unsigned int sysctl_sched_shares_thresh = 4;
  692. /*
  693. * period over which we measure -rt task cpu usage in us.
  694. * default: 1s
  695. */
  696. unsigned int sysctl_sched_rt_period = 1000000;
  697. static __read_mostly int scheduler_running;
  698. /*
  699. * part of the period that we allow rt tasks to run in us.
  700. * default: 0.95s
  701. */
  702. int sysctl_sched_rt_runtime = 950000;
  703. static inline u64 global_rt_period(void)
  704. {
  705. return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
  706. }
  707. static inline u64 global_rt_runtime(void)
  708. {
  709. if (sysctl_sched_rt_runtime < 0)
  710. return RUNTIME_INF;
  711. return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
  712. }
  713. #ifndef prepare_arch_switch
  714. # define prepare_arch_switch(next) do { } while (0)
  715. #endif
  716. #ifndef finish_arch_switch
  717. # define finish_arch_switch(prev) do { } while (0)
  718. #endif
  719. static inline int task_current(struct rq *rq, struct task_struct *p)
  720. {
  721. return rq->curr == p;
  722. }
  723. #ifndef __ARCH_WANT_UNLOCKED_CTXSW
  724. static inline int task_running(struct rq *rq, struct task_struct *p)
  725. {
  726. return task_current(rq, p);
  727. }
  728. static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
  729. {
  730. }
  731. static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
  732. {
  733. #ifdef CONFIG_DEBUG_SPINLOCK
  734. /* this is a valid case when another task releases the spinlock */
  735. rq->lock.owner = current;
  736. #endif
  737. /*
  738. * If we are tracking spinlock dependencies then we have to
  739. * fix up the runqueue lock - which gets 'carried over' from
  740. * prev into current:
  741. */
  742. spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
  743. spin_unlock_irq(&rq->lock);
  744. }
  745. #else /* __ARCH_WANT_UNLOCKED_CTXSW */
  746. static inline int task_running(struct rq *rq, struct task_struct *p)
  747. {
  748. #ifdef CONFIG_SMP
  749. return p->oncpu;
  750. #else
  751. return task_current(rq, p);
  752. #endif
  753. }
  754. static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
  755. {
  756. #ifdef CONFIG_SMP
  757. /*
  758. * We can optimise this out completely for !SMP, because the
  759. * SMP rebalancing from interrupt is the only thing that cares
  760. * here.
  761. */
  762. next->oncpu = 1;
  763. #endif
  764. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  765. spin_unlock_irq(&rq->lock);
  766. #else
  767. spin_unlock(&rq->lock);
  768. #endif
  769. }
  770. static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
  771. {
  772. #ifdef CONFIG_SMP
  773. /*
  774. * After ->oncpu is cleared, the task can be moved to a different CPU.
  775. * We must ensure this doesn't happen until the switch is completely
  776. * finished.
  777. */
  778. smp_wmb();
  779. prev->oncpu = 0;
  780. #endif
  781. #ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  782. local_irq_enable();
  783. #endif
  784. }
  785. #endif /* __ARCH_WANT_UNLOCKED_CTXSW */
  786. /*
  787. * __task_rq_lock - lock the runqueue a given task resides on.
  788. * Must be called interrupts disabled.
  789. */
  790. static inline struct rq *__task_rq_lock(struct task_struct *p)
  791. __acquires(rq->lock)
  792. {
  793. for (;;) {
  794. struct rq *rq = task_rq(p);
  795. spin_lock(&rq->lock);
  796. if (likely(rq == task_rq(p)))
  797. return rq;
  798. spin_unlock(&rq->lock);
  799. }
  800. }
  801. /*
  802. * task_rq_lock - lock the runqueue a given task resides on and disable
  803. * interrupts. Note the ordering: we can safely lookup the task_rq without
  804. * explicitly disabling preemption.
  805. */
  806. static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
  807. __acquires(rq->lock)
  808. {
  809. struct rq *rq;
  810. for (;;) {
  811. local_irq_save(*flags);
  812. rq = task_rq(p);
  813. spin_lock(&rq->lock);
  814. if (likely(rq == task_rq(p)))
  815. return rq;
  816. spin_unlock_irqrestore(&rq->lock, *flags);
  817. }
  818. }
  819. void task_rq_unlock_wait(struct task_struct *p)
  820. {
  821. struct rq *rq = task_rq(p);
  822. smp_mb(); /* spin-unlock-wait is not a full memory barrier */
  823. spin_unlock_wait(&rq->lock);
  824. }
  825. static void __task_rq_unlock(struct rq *rq)
  826. __releases(rq->lock)
  827. {
  828. spin_unlock(&rq->lock);
  829. }
  830. static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
  831. __releases(rq->lock)
  832. {
  833. spin_unlock_irqrestore(&rq->lock, *flags);
  834. }
  835. /*
  836. * this_rq_lock - lock this runqueue and disable interrupts.
  837. */
  838. static struct rq *this_rq_lock(void)
  839. __acquires(rq->lock)
  840. {
  841. struct rq *rq;
  842. local_irq_disable();
  843. rq = this_rq();
  844. spin_lock(&rq->lock);
  845. return rq;
  846. }
  847. #ifdef CONFIG_SCHED_HRTICK
  848. /*
  849. * Use HR-timers to deliver accurate preemption points.
  850. *
  851. * Its all a bit involved since we cannot program an hrt while holding the
  852. * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
  853. * reschedule event.
  854. *
  855. * When we get rescheduled we reprogram the hrtick_timer outside of the
  856. * rq->lock.
  857. */
  858. /*
  859. * Use hrtick when:
  860. * - enabled by features
  861. * - hrtimer is actually high res
  862. */
  863. static inline int hrtick_enabled(struct rq *rq)
  864. {
  865. if (!sched_feat(HRTICK))
  866. return 0;
  867. if (!cpu_active(cpu_of(rq)))
  868. return 0;
  869. return hrtimer_is_hres_active(&rq->hrtick_timer);
  870. }
  871. static void hrtick_clear(struct rq *rq)
  872. {
  873. if (hrtimer_active(&rq->hrtick_timer))
  874. hrtimer_cancel(&rq->hrtick_timer);
  875. }
  876. /*
  877. * High-resolution timer tick.
  878. * Runs from hardirq context with interrupts disabled.
  879. */
  880. static enum hrtimer_restart hrtick(struct hrtimer *timer)
  881. {
  882. struct rq *rq = container_of(timer, struct rq, hrtick_timer);
  883. WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
  884. spin_lock(&rq->lock);
  885. update_rq_clock(rq);
  886. rq->curr->sched_class->task_tick(rq, rq->curr, 1);
  887. spin_unlock(&rq->lock);
  888. return HRTIMER_NORESTART;
  889. }
  890. #ifdef CONFIG_SMP
  891. /*
  892. * called from hardirq (IPI) context
  893. */
  894. static void __hrtick_start(void *arg)
  895. {
  896. struct rq *rq = arg;
  897. spin_lock(&rq->lock);
  898. hrtimer_restart(&rq->hrtick_timer);
  899. rq->hrtick_csd_pending = 0;
  900. spin_unlock(&rq->lock);
  901. }
  902. /*
  903. * Called to set the hrtick timer state.
  904. *
  905. * called with rq->lock held and irqs disabled
  906. */
  907. static void hrtick_start(struct rq *rq, u64 delay)
  908. {
  909. struct hrtimer *timer = &rq->hrtick_timer;
  910. ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
  911. hrtimer_set_expires(timer, time);
  912. if (rq == this_rq()) {
  913. hrtimer_restart(timer);
  914. } else if (!rq->hrtick_csd_pending) {
  915. __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd);
  916. rq->hrtick_csd_pending = 1;
  917. }
  918. }
  919. static int
  920. hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
  921. {
  922. int cpu = (int)(long)hcpu;
  923. switch (action) {
  924. case CPU_UP_CANCELED:
  925. case CPU_UP_CANCELED_FROZEN:
  926. case CPU_DOWN_PREPARE:
  927. case CPU_DOWN_PREPARE_FROZEN:
  928. case CPU_DEAD:
  929. case CPU_DEAD_FROZEN:
  930. hrtick_clear(cpu_rq(cpu));
  931. return NOTIFY_OK;
  932. }
  933. return NOTIFY_DONE;
  934. }
  935. static __init void init_hrtick(void)
  936. {
  937. hotcpu_notifier(hotplug_hrtick, 0);
  938. }
  939. #else
  940. /*
  941. * Called to set the hrtick timer state.
  942. *
  943. * called with rq->lock held and irqs disabled
  944. */
  945. static void hrtick_start(struct rq *rq, u64 delay)
  946. {
  947. hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL);
  948. }
  949. static inline void init_hrtick(void)
  950. {
  951. }
  952. #endif /* CONFIG_SMP */
  953. static void init_rq_hrtick(struct rq *rq)
  954. {
  955. #ifdef CONFIG_SMP
  956. rq->hrtick_csd_pending = 0;
  957. rq->hrtick_csd.flags = 0;
  958. rq->hrtick_csd.func = __hrtick_start;
  959. rq->hrtick_csd.info = rq;
  960. #endif
  961. hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  962. rq->hrtick_timer.function = hrtick;
  963. }
  964. #else /* CONFIG_SCHED_HRTICK */
  965. static inline void hrtick_clear(struct rq *rq)
  966. {
  967. }
  968. static inline void init_rq_hrtick(struct rq *rq)
  969. {
  970. }
  971. static inline void init_hrtick(void)
  972. {
  973. }
  974. #endif /* CONFIG_SCHED_HRTICK */
  975. /*
  976. * resched_task - mark a task 'to be rescheduled now'.
  977. *
  978. * On UP this means the setting of the need_resched flag, on SMP it
  979. * might also involve a cross-CPU call to trigger the scheduler on
  980. * the target CPU.
  981. */
  982. #ifdef CONFIG_SMP
  983. #ifndef tsk_is_polling
  984. #define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
  985. #endif
  986. static void resched_task(struct task_struct *p)
  987. {
  988. int cpu;
  989. assert_spin_locked(&task_rq(p)->lock);
  990. if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
  991. return;
  992. set_tsk_thread_flag(p, TIF_NEED_RESCHED);
  993. cpu = task_cpu(p);
  994. if (cpu == smp_processor_id())
  995. return;
  996. /* NEED_RESCHED must be visible before we test polling */
  997. smp_mb();
  998. if (!tsk_is_polling(p))
  999. smp_send_reschedule(cpu);
  1000. }
  1001. static void resched_cpu(int cpu)
  1002. {
  1003. struct rq *rq = cpu_rq(cpu);
  1004. unsigned long flags;
  1005. if (!spin_trylock_irqsave(&rq->lock, flags))
  1006. return;
  1007. resched_task(cpu_curr(cpu));
  1008. spin_unlock_irqrestore(&rq->lock, flags);
  1009. }
  1010. #ifdef CONFIG_NO_HZ
  1011. /*
  1012. * When add_timer_on() enqueues a timer into the timer wheel of an
  1013. * idle CPU then this timer might expire before the next timer event
  1014. * which is scheduled to wake up that CPU. In case of a completely
  1015. * idle system the next event might even be infinite time into the
  1016. * future. wake_up_idle_cpu() ensures that the CPU is woken up and
  1017. * leaves the inner idle loop so the newly added timer is taken into
  1018. * account when the CPU goes back to idle and evaluates the timer
  1019. * wheel for the next timer event.
  1020. */
  1021. void wake_up_idle_cpu(int cpu)
  1022. {
  1023. struct rq *rq = cpu_rq(cpu);
  1024. if (cpu == smp_processor_id())
  1025. return;
  1026. /*
  1027. * This is safe, as this function is called with the timer
  1028. * wheel base lock of (cpu) held. When the CPU is on the way
  1029. * to idle and has not yet set rq->curr to idle then it will
  1030. * be serialized on the timer wheel base lock and take the new
  1031. * timer into account automatically.
  1032. */
  1033. if (rq->curr != rq->idle)
  1034. return;
  1035. /*
  1036. * We can set TIF_RESCHED on the idle task of the other CPU
  1037. * lockless. The worst case is that the other CPU runs the
  1038. * idle task through an additional NOOP schedule()
  1039. */
  1040. set_tsk_thread_flag(rq->idle, TIF_NEED_RESCHED);
  1041. /* NEED_RESCHED must be visible before we test polling */
  1042. smp_mb();
  1043. if (!tsk_is_polling(rq->idle))
  1044. smp_send_reschedule(cpu);
  1045. }
  1046. #endif /* CONFIG_NO_HZ */
  1047. #else /* !CONFIG_SMP */
  1048. static void resched_task(struct task_struct *p)
  1049. {
  1050. assert_spin_locked(&task_rq(p)->lock);
  1051. set_tsk_need_resched(p);
  1052. }
  1053. #endif /* CONFIG_SMP */
  1054. #if BITS_PER_LONG == 32
  1055. # define WMULT_CONST (~0UL)
  1056. #else
  1057. # define WMULT_CONST (1UL << 32)
  1058. #endif
  1059. #define WMULT_SHIFT 32
  1060. /*
  1061. * Shift right and round:
  1062. */
  1063. #define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
  1064. /*
  1065. * delta *= weight / lw
  1066. */
  1067. static unsigned long
  1068. calc_delta_mine(unsigned long delta_exec, unsigned long weight,
  1069. struct load_weight *lw)
  1070. {
  1071. u64 tmp;
  1072. if (!lw->inv_weight) {
  1073. if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
  1074. lw->inv_weight = 1;
  1075. else
  1076. lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
  1077. / (lw->weight+1);
  1078. }
  1079. tmp = (u64)delta_exec * weight;
  1080. /*
  1081. * Check whether we'd overflow the 64-bit multiplication:
  1082. */
  1083. if (unlikely(tmp > WMULT_CONST))
  1084. tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
  1085. WMULT_SHIFT/2);
  1086. else
  1087. tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
  1088. return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
  1089. }
  1090. static inline void update_load_add(struct load_weight *lw, unsigned long inc)
  1091. {
  1092. lw->weight += inc;
  1093. lw->inv_weight = 0;
  1094. }
  1095. static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
  1096. {
  1097. lw->weight -= dec;
  1098. lw->inv_weight = 0;
  1099. }
  1100. /*
  1101. * To aid in avoiding the subversion of "niceness" due to uneven distribution
  1102. * of tasks with abnormal "nice" values across CPUs the contribution that
  1103. * each task makes to its run queue's load is weighted according to its
  1104. * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
  1105. * scaled version of the new time slice allocation that they receive on time
  1106. * slice expiry etc.
  1107. */
  1108. #define WEIGHT_IDLEPRIO 2
  1109. #define WMULT_IDLEPRIO (1 << 31)
  1110. /*
  1111. * Nice levels are multiplicative, with a gentle 10% change for every
  1112. * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
  1113. * nice 1, it will get ~10% less CPU time than another CPU-bound task
  1114. * that remained on nice 0.
  1115. *
  1116. * The "10% effect" is relative and cumulative: from _any_ nice level,
  1117. * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
  1118. * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
  1119. * If a task goes up by ~10% and another task goes down by ~10% then
  1120. * the relative distance between them is ~25%.)
  1121. */
  1122. static const int prio_to_weight[40] = {
  1123. /* -20 */ 88761, 71755, 56483, 46273, 36291,
  1124. /* -15 */ 29154, 23254, 18705, 14949, 11916,
  1125. /* -10 */ 9548, 7620, 6100, 4904, 3906,
  1126. /* -5 */ 3121, 2501, 1991, 1586, 1277,
  1127. /* 0 */ 1024, 820, 655, 526, 423,
  1128. /* 5 */ 335, 272, 215, 172, 137,
  1129. /* 10 */ 110, 87, 70, 56, 45,
  1130. /* 15 */ 36, 29, 23, 18, 15,
  1131. };
  1132. /*
  1133. * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
  1134. *
  1135. * In cases where the weight does not change often, we can use the
  1136. * precalculated inverse to speed up arithmetics by turning divisions
  1137. * into multiplications:
  1138. */
  1139. static const u32 prio_to_wmult[40] = {
  1140. /* -20 */ 48388, 59856, 76040, 92818, 118348,
  1141. /* -15 */ 147320, 184698, 229616, 287308, 360437,
  1142. /* -10 */ 449829, 563644, 704093, 875809, 1099582,
  1143. /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
  1144. /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
  1145. /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
  1146. /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
  1147. /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
  1148. };
  1149. static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);
  1150. /*
  1151. * runqueue iterator, to support SMP load-balancing between different
  1152. * scheduling classes, without having to expose their internal data
  1153. * structures to the load-balancing proper:
  1154. */
  1155. struct rq_iterator {
  1156. void *arg;
  1157. struct task_struct *(*start)(void *);
  1158. struct task_struct *(*next)(void *);
  1159. };
  1160. #ifdef CONFIG_SMP
  1161. static unsigned long
  1162. balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
  1163. unsigned long max_load_move, struct sched_domain *sd,
  1164. enum cpu_idle_type idle, int *all_pinned,
  1165. int *this_best_prio, struct rq_iterator *iterator);
  1166. static int
  1167. iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
  1168. struct sched_domain *sd, enum cpu_idle_type idle,
  1169. struct rq_iterator *iterator);
  1170. #endif
  1171. #ifdef CONFIG_CGROUP_CPUACCT
  1172. static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
  1173. #else
  1174. static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
  1175. #endif
  1176. static inline void inc_cpu_load(struct rq *rq, unsigned long load)
  1177. {
  1178. update_load_add(&rq->load, load);
  1179. }
  1180. static inline void dec_cpu_load(struct rq *rq, unsigned long load)
  1181. {
  1182. update_load_sub(&rq->load, load);
  1183. }
  1184. #if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) || defined(CONFIG_RT_GROUP_SCHED)
  1185. typedef int (*tg_visitor)(struct task_group *, void *);
  1186. /*
  1187. * Iterate the full tree, calling @down when first entering a node and @up when
  1188. * leaving it for the final time.
  1189. */
  1190. static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
  1191. {
  1192. struct task_group *parent, *child;
  1193. int ret;
  1194. rcu_read_lock();
  1195. parent = &root_task_group;
  1196. down:
  1197. ret = (*down)(parent, data);
  1198. if (ret)
  1199. goto out_unlock;
  1200. list_for_each_entry_rcu(child, &parent->children, siblings) {
  1201. parent = child;
  1202. goto down;
  1203. up:
  1204. continue;
  1205. }
  1206. ret = (*up)(parent, data);
  1207. if (ret)
  1208. goto out_unlock;
  1209. child = parent;
  1210. parent = parent->parent;
  1211. if (parent)
  1212. goto up;
  1213. out_unlock:
  1214. rcu_read_unlock();
  1215. return ret;
  1216. }
  1217. static int tg_nop(struct task_group *tg, void *data)
  1218. {
  1219. return 0;
  1220. }
  1221. #endif
  1222. #ifdef CONFIG_SMP
  1223. static unsigned long source_load(int cpu, int type);
  1224. static unsigned long target_load(int cpu, int type);
  1225. static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
  1226. static unsigned long cpu_avg_load_per_task(int cpu)
  1227. {
  1228. struct rq *rq = cpu_rq(cpu);
  1229. unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
  1230. if (nr_running)
  1231. rq->avg_load_per_task = rq->load.weight / nr_running;
  1232. else
  1233. rq->avg_load_per_task = 0;
  1234. return rq->avg_load_per_task;
  1235. }
  1236. #ifdef CONFIG_FAIR_GROUP_SCHED
  1237. static void __set_se_shares(struct sched_entity *se, unsigned long shares);
  1238. /*
  1239. * Calculate and set the cpu's group shares.
  1240. */
  1241. static void
  1242. update_group_shares_cpu(struct task_group *tg, int cpu,
  1243. unsigned long sd_shares, unsigned long sd_rq_weight)
  1244. {
  1245. unsigned long shares;
  1246. unsigned long rq_weight;
  1247. if (!tg->se[cpu])
  1248. return;
  1249. rq_weight = tg->cfs_rq[cpu]->rq_weight;
  1250. /*
  1251. * \Sum shares * rq_weight
  1252. * shares = -----------------------
  1253. * \Sum rq_weight
  1254. *
  1255. */
  1256. shares = (sd_shares * rq_weight) / sd_rq_weight;
  1257. shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
  1258. if (abs(shares - tg->se[cpu]->load.weight) >
  1259. sysctl_sched_shares_thresh) {
  1260. struct rq *rq = cpu_rq(cpu);
  1261. unsigned long flags;
  1262. spin_lock_irqsave(&rq->lock, flags);
  1263. tg->cfs_rq[cpu]->shares = shares;
  1264. __set_se_shares(tg->se[cpu], shares);
  1265. spin_unlock_irqrestore(&rq->lock, flags);
  1266. }
  1267. }
  1268. /*
  1269. * Re-compute the task group their per cpu shares over the given domain.
  1270. * This needs to be done in a bottom-up fashion because the rq weight of a
  1271. * parent group depends on the shares of its child groups.
  1272. */
  1273. static int tg_shares_up(struct task_group *tg, void *data)
  1274. {
  1275. unsigned long weight, rq_weight = 0;
  1276. unsigned long shares = 0;
  1277. struct sched_domain *sd = data;
  1278. int i;
  1279. for_each_cpu(i, sched_domain_span(sd)) {
  1280. /*
  1281. * If there are currently no tasks on the cpu pretend there
  1282. * is one of average load so that when a new task gets to
  1283. * run here it will not get delayed by group starvation.
  1284. */
  1285. weight = tg->cfs_rq[i]->load.weight;
  1286. if (!weight)
  1287. weight = NICE_0_LOAD;
  1288. tg->cfs_rq[i]->rq_weight = weight;
  1289. rq_weight += weight;
  1290. shares += tg->cfs_rq[i]->shares;
  1291. }
  1292. if ((!shares && rq_weight) || shares > tg->shares)
  1293. shares = tg->shares;
  1294. if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
  1295. shares = tg->shares;
  1296. for_each_cpu(i, sched_domain_span(sd))
  1297. update_group_shares_cpu(tg, i, shares, rq_weight);
  1298. return 0;
  1299. }
  1300. /*
  1301. * Compute the cpu's hierarchical load factor for each task group.
  1302. * This needs to be done in a top-down fashion because the load of a child
  1303. * group is a fraction of its parents load.
  1304. */
  1305. static int tg_load_down(struct task_group *tg, void *data)
  1306. {
  1307. unsigned long load;
  1308. long cpu = (long)data;
  1309. if (!tg->parent) {
  1310. load = cpu_rq(cpu)->load.weight;
  1311. } else {
  1312. load = tg->parent->cfs_rq[cpu]->h_load;
  1313. load *= tg->cfs_rq[cpu]->shares;
  1314. load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
  1315. }
  1316. tg->cfs_rq[cpu]->h_load = load;
  1317. return 0;
  1318. }
  1319. static void update_shares(struct sched_domain *sd)
  1320. {
  1321. u64 now = cpu_clock(raw_smp_processor_id());
  1322. s64 elapsed = now - sd->last_update;
  1323. if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
  1324. sd->last_update = now;
  1325. walk_tg_tree(tg_nop, tg_shares_up, sd);
  1326. }
  1327. }
  1328. static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
  1329. {
  1330. spin_unlock(&rq->lock);
  1331. update_shares(sd);
  1332. spin_lock(&rq->lock);
  1333. }
  1334. static void update_h_load(long cpu)
  1335. {
  1336. walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
  1337. }
  1338. #else
  1339. static inline void update_shares(struct sched_domain *sd)
  1340. {
  1341. }
  1342. static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
  1343. {
  1344. }
  1345. #endif
  1346. /*
  1347. * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
  1348. */
  1349. static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1350. __releases(this_rq->lock)
  1351. __acquires(busiest->lock)
  1352. __acquires(this_rq->lock)
  1353. {
  1354. int ret = 0;
  1355. if (unlikely(!irqs_disabled())) {
  1356. /* printk() doesn't work good under rq->lock */
  1357. spin_unlock(&this_rq->lock);
  1358. BUG_ON(1);
  1359. }
  1360. if (unlikely(!spin_trylock(&busiest->lock))) {
  1361. if (busiest < this_rq) {
  1362. spin_unlock(&this_rq->lock);
  1363. spin_lock(&busiest->lock);
  1364. spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
  1365. ret = 1;
  1366. } else
  1367. spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
  1368. }
  1369. return ret;
  1370. }
  1371. static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
  1372. __releases(busiest->lock)
  1373. {
  1374. spin_unlock(&busiest->lock);
  1375. lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
  1376. }
  1377. #endif
  1378. #ifdef CONFIG_FAIR_GROUP_SCHED
  1379. static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
  1380. {
  1381. #ifdef CONFIG_SMP
  1382. cfs_rq->shares = shares;
  1383. #endif
  1384. }
  1385. #endif
  1386. #include "sched_stats.h"
  1387. #include "sched_idletask.c"
  1388. #include "sched_fair.c"
  1389. #include "sched_rt.c"
  1390. #ifdef CONFIG_SCHED_DEBUG
  1391. # include "sched_debug.c"
  1392. #endif
  1393. #define sched_class_highest (&rt_sched_class)
  1394. #define for_each_class(class) \
  1395. for (class = sched_class_highest; class; class = class->next)
  1396. static void inc_nr_running(struct rq *rq)
  1397. {
  1398. rq->nr_running++;
  1399. }
  1400. static void dec_nr_running(struct rq *rq)
  1401. {
  1402. rq->nr_running--;
  1403. }
  1404. static void set_load_weight(struct task_struct *p)
  1405. {
  1406. if (task_has_rt_policy(p)) {
  1407. p->se.load.weight = prio_to_weight[0] * 2;
  1408. p->se.load.inv_weight = prio_to_wmult[0] >> 1;
  1409. return;
  1410. }
  1411. /*
  1412. * SCHED_IDLE tasks get minimal weight:
  1413. */
  1414. if (p->policy == SCHED_IDLE) {
  1415. p->se.load.weight = WEIGHT_IDLEPRIO;
  1416. p->se.load.inv_weight = WMULT_IDLEPRIO;
  1417. return;
  1418. }
  1419. p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
  1420. p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
  1421. }
  1422. static void update_avg(u64 *avg, u64 sample)
  1423. {
  1424. s64 diff = sample - *avg;
  1425. *avg += diff >> 3;
  1426. }
  1427. static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
  1428. {
  1429. sched_info_queued(p);
  1430. p->sched_class->enqueue_task(rq, p, wakeup);
  1431. p->se.on_rq = 1;
  1432. }
  1433. static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
  1434. {
  1435. if (sleep && p->se.last_wakeup) {
  1436. update_avg(&p->se.avg_overlap,
  1437. p->se.sum_exec_runtime - p->se.last_wakeup);
  1438. p->se.last_wakeup = 0;
  1439. }
  1440. sched_info_dequeued(p);
  1441. p->sched_class->dequeue_task(rq, p, sleep);
  1442. p->se.on_rq = 0;
  1443. }
  1444. /*
  1445. * __normal_prio - return the priority that is based on the static prio
  1446. */
  1447. static inline int __normal_prio(struct task_struct *p)
  1448. {
  1449. return p->static_prio;
  1450. }
  1451. /*
  1452. * Calculate the expected normal priority: i.e. priority
  1453. * without taking RT-inheritance into account. Might be
  1454. * boosted by interactivity modifiers. Changes upon fork,
  1455. * setprio syscalls, and whenever the interactivity
  1456. * estimator recalculates.
  1457. */
  1458. static inline int normal_prio(struct task_struct *p)
  1459. {
  1460. int prio;
  1461. if (task_has_rt_policy(p))
  1462. prio = MAX_RT_PRIO-1 - p->rt_priority;
  1463. else
  1464. prio = __normal_prio(p);
  1465. return prio;
  1466. }
  1467. /*
  1468. * Calculate the current priority, i.e. the priority
  1469. * taken into account by the scheduler. This value might
  1470. * be boosted by RT tasks, or might be boosted by
  1471. * interactivity modifiers. Will be RT if the task got
  1472. * RT-boosted. If not then it returns p->normal_prio.
  1473. */
  1474. static int effective_prio(struct task_struct *p)
  1475. {
  1476. p->normal_prio = normal_prio(p);
  1477. /*
  1478. * If we are RT tasks or we were boosted to RT priority,
  1479. * keep the priority unchanged. Otherwise, update priority
  1480. * to the normal priority:
  1481. */
  1482. if (!rt_prio(p->prio))
  1483. return p->normal_prio;
  1484. return p->prio;
  1485. }
  1486. /*
  1487. * activate_task - move a task to the runqueue.
  1488. */
  1489. static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
  1490. {
  1491. if (task_contributes_to_load(p))
  1492. rq->nr_uninterruptible--;
  1493. enqueue_task(rq, p, wakeup);
  1494. inc_nr_running(rq);
  1495. }
  1496. /*
  1497. * deactivate_task - remove a task from the runqueue.
  1498. */
  1499. static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
  1500. {
  1501. if (task_contributes_to_load(p))
  1502. rq->nr_uninterruptible++;
  1503. dequeue_task(rq, p, sleep);
  1504. dec_nr_running(rq);
  1505. }
  1506. /**
  1507. * task_curr - is this task currently executing on a CPU?
  1508. * @p: the task in question.
  1509. */
  1510. inline int task_curr(const struct task_struct *p)
  1511. {
  1512. return cpu_curr(task_cpu(p)) == p;
  1513. }
  1514. static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
  1515. {
  1516. set_task_rq(p, cpu);
  1517. #ifdef CONFIG_SMP
  1518. /*
  1519. * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
  1520. * successfuly executed on another CPU. We must ensure that updates of
  1521. * per-task data have been completed by this moment.
  1522. */
  1523. smp_wmb();
  1524. task_thread_info(p)->cpu = cpu;
  1525. #endif
  1526. }
  1527. static inline void check_class_changed(struct rq *rq, struct task_struct *p,
  1528. const struct sched_class *prev_class,
  1529. int oldprio, int running)
  1530. {
  1531. if (prev_class != p->sched_class) {
  1532. if (prev_class->switched_from)
  1533. prev_class->switched_from(rq, p, running);
  1534. p->sched_class->switched_to(rq, p, running);
  1535. } else
  1536. p->sched_class->prio_changed(rq, p, oldprio, running);
  1537. }
  1538. #ifdef CONFIG_SMP
  1539. /* Used instead of source_load when we know the type == 0 */
  1540. static unsigned long weighted_cpuload(const int cpu)
  1541. {
  1542. return cpu_rq(cpu)->load.weight;
  1543. }
  1544. /*
  1545. * Is this task likely cache-hot:
  1546. */
  1547. static int
  1548. task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
  1549. {
  1550. s64 delta;
  1551. /*
  1552. * Buddy candidates are cache hot:
  1553. */
  1554. if (sched_feat(CACHE_HOT_BUDDY) &&
  1555. (&p->se == cfs_rq_of(&p->se)->next ||
  1556. &p->se == cfs_rq_of(&p->se)->last))
  1557. return 1;
  1558. if (p->sched_class != &fair_sched_class)
  1559. return 0;
  1560. if (sysctl_sched_migration_cost == -1)
  1561. return 1;
  1562. if (sysctl_sched_migration_cost == 0)
  1563. return 0;
  1564. delta = now - p->se.exec_start;
  1565. return delta < (s64)sysctl_sched_migration_cost;
  1566. }
  1567. void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
  1568. {
  1569. int old_cpu = task_cpu(p);
  1570. struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
  1571. struct cfs_rq *old_cfsrq = task_cfs_rq(p),
  1572. *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
  1573. u64 clock_offset;
  1574. clock_offset = old_rq->clock - new_rq->clock;
  1575. trace_sched_migrate_task(p, task_cpu(p), new_cpu);
  1576. #ifdef CONFIG_SCHEDSTATS
  1577. if (p->se.wait_start)
  1578. p->se.wait_start -= clock_offset;
  1579. if (p->se.sleep_start)
  1580. p->se.sleep_start -= clock_offset;
  1581. if (p->se.block_start)
  1582. p->se.block_start -= clock_offset;
  1583. if (old_cpu != new_cpu) {
  1584. schedstat_inc(p, se.nr_migrations);
  1585. if (task_hot(p, old_rq->clock, NULL))
  1586. schedstat_inc(p, se.nr_forced2_migrations);
  1587. }
  1588. #endif
  1589. p->se.vruntime -= old_cfsrq->min_vruntime -
  1590. new_cfsrq->min_vruntime;
  1591. __set_task_cpu(p, new_cpu);
  1592. }
  1593. struct migration_req {
  1594. struct list_head list;
  1595. struct task_struct *task;
  1596. int dest_cpu;
  1597. struct completion done;
  1598. };
  1599. /*
  1600. * The task's runqueue lock must be held.
  1601. * Returns true if you have to wait for migration thread.
  1602. */
  1603. static int
  1604. migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
  1605. {
  1606. struct rq *rq = task_rq(p);
  1607. /*
  1608. * If the task is not on a runqueue (and not running), then
  1609. * it is sufficient to simply update the task's cpu field.
  1610. */
  1611. if (!p->se.on_rq && !task_running(rq, p)) {
  1612. set_task_cpu(p, dest_cpu);
  1613. return 0;
  1614. }
  1615. init_completion(&req->done);
  1616. req->task = p;
  1617. req->dest_cpu = dest_cpu;
  1618. list_add(&req->list, &rq->migration_queue);
  1619. return 1;
  1620. }
  1621. /*
  1622. * wait_task_inactive - wait for a thread to unschedule.
  1623. *
  1624. * If @match_state is nonzero, it's the @p->state value just checked and
  1625. * not expected to change. If it changes, i.e. @p might have woken up,
  1626. * then return zero. When we succeed in waiting for @p to be off its CPU,
  1627. * we return a positive number (its total switch count). If a second call
  1628. * a short while later returns the same number, the caller can be sure that
  1629. * @p has remained unscheduled the whole time.
  1630. *
  1631. * The caller must ensure that the task *will* unschedule sometime soon,
  1632. * else this function might spin for a *long* time. This function can't
  1633. * be called with interrupts off, or it may introduce deadlock with
  1634. * smp_call_function() if an IPI is sent by the same process we are
  1635. * waiting to become inactive.
  1636. */
  1637. unsigned long wait_task_inactive(struct task_struct *p, long match_state)
  1638. {
  1639. unsigned long flags;
  1640. int running, on_rq;
  1641. unsigned long ncsw;
  1642. struct rq *rq;
  1643. for (;;) {
  1644. /*
  1645. * We do the initial early heuristics without holding
  1646. * any task-queue locks at all. We'll only try to get
  1647. * the runqueue lock when things look like they will
  1648. * work out!
  1649. */
  1650. rq = task_rq(p);
  1651. /*
  1652. * If the task is actively running on another CPU
  1653. * still, just relax and busy-wait without holding
  1654. * any locks.
  1655. *
  1656. * NOTE! Since we don't hold any locks, it's not
  1657. * even sure that "rq" stays as the right runqueue!
  1658. * But we don't care, since "task_running()" will
  1659. * return false if the runqueue has changed and p
  1660. * is actually now running somewhere else!
  1661. */
  1662. while (task_running(rq, p)) {
  1663. if (match_state && unlikely(p->state != match_state))
  1664. return 0;
  1665. cpu_relax();
  1666. }
  1667. /*
  1668. * Ok, time to look more closely! We need the rq
  1669. * lock now, to be *sure*. If we're wrong, we'll
  1670. * just go back and repeat.
  1671. */
  1672. rq = task_rq_lock(p, &flags);
  1673. trace_sched_wait_task(rq, p);
  1674. running = task_running(rq, p);
  1675. on_rq = p->se.on_rq;
  1676. ncsw = 0;
  1677. if (!match_state || p->state == match_state)
  1678. ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
  1679. task_rq_unlock(rq, &flags);
  1680. /*
  1681. * If it changed from the expected state, bail out now.
  1682. */
  1683. if (unlikely(!ncsw))
  1684. break;
  1685. /*
  1686. * Was it really running after all now that we
  1687. * checked with the proper locks actually held?
  1688. *
  1689. * Oops. Go back and try again..
  1690. */
  1691. if (unlikely(running)) {
  1692. cpu_relax();
  1693. continue;
  1694. }
  1695. /*
  1696. * It's not enough that it's not actively running,
  1697. * it must be off the runqueue _entirely_, and not
  1698. * preempted!
  1699. *
  1700. * So if it wa still runnable (but just not actively
  1701. * running right now), it's preempted, and we should
  1702. * yield - it could be a while.
  1703. */
  1704. if (unlikely(on_rq)) {
  1705. schedule_timeout_uninterruptible(1);
  1706. continue;
  1707. }
  1708. /*
  1709. * Ahh, all good. It wasn't running, and it wasn't
  1710. * runnable, which means that it will never become
  1711. * running in the future either. We're all done!
  1712. */
  1713. break;
  1714. }
  1715. return ncsw;
  1716. }
  1717. /***
  1718. * kick_process - kick a running thread to enter/exit the kernel
  1719. * @p: the to-be-kicked thread
  1720. *
  1721. * Cause a process which is running on another CPU to enter
  1722. * kernel-mode, without any delay. (to get signals handled.)
  1723. *
  1724. * NOTE: this function doesnt have to take the runqueue lock,
  1725. * because all it wants to ensure is that the remote task enters
  1726. * the kernel. If the IPI races and the task has been migrated
  1727. * to another CPU then no harm is done and the purpose has been
  1728. * achieved as well.
  1729. */
  1730. void kick_process(struct task_struct *p)
  1731. {
  1732. int cpu;
  1733. preempt_disable();
  1734. cpu = task_cpu(p);
  1735. if ((cpu != smp_processor_id()) && task_curr(p))
  1736. smp_send_reschedule(cpu);
  1737. preempt_enable();
  1738. }
  1739. /*
  1740. * Return a low guess at the load of a migration-source cpu weighted
  1741. * according to the scheduling class and "nice" value.
  1742. *
  1743. * We want to under-estimate the load of migration sources, to
  1744. * balance conservatively.
  1745. */
  1746. static unsigned long source_load(int cpu, int type)
  1747. {
  1748. struct rq *rq = cpu_rq(cpu);
  1749. unsigned long total = weighted_cpuload(cpu);
  1750. if (type == 0 || !sched_feat(LB_BIAS))
  1751. return total;
  1752. return min(rq->cpu_load[type-1], total);
  1753. }
  1754. /*
  1755. * Return a high guess at the load of a migration-target cpu weighted
  1756. * according to the scheduling class and "nice" value.
  1757. */
  1758. static unsigned long target_load(int cpu, int type)
  1759. {
  1760. struct rq *rq = cpu_rq(cpu);
  1761. unsigned long total = weighted_cpuload(cpu);
  1762. if (type == 0 || !sched_feat(LB_BIAS))
  1763. return total;
  1764. return max(rq->cpu_load[type-1], total);
  1765. }
  1766. /*
  1767. * find_idlest_group finds and returns the least busy CPU group within the
  1768. * domain.
  1769. */
  1770. static struct sched_group *
  1771. find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
  1772. {
  1773. struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
  1774. unsigned long min_load = ULONG_MAX, this_load = 0;
  1775. int load_idx = sd->forkexec_idx;
  1776. int imbalance = 100 + (sd->imbalance_pct-100)/2;
  1777. do {
  1778. unsigned long load, avg_load;
  1779. int local_group;
  1780. int i;
  1781. /* Skip over this group if it has no CPUs allowed */
  1782. if (!cpumask_intersects(sched_group_cpus(group),
  1783. &p->cpus_allowed))
  1784. continue;
  1785. local_group = cpumask_test_cpu(this_cpu,
  1786. sched_group_cpus(group));
  1787. /* Tally up the load of all CPUs in the group */
  1788. avg_load = 0;
  1789. for_each_cpu(i, sched_group_cpus(group)) {
  1790. /* Bias balancing toward cpus of our domain */
  1791. if (local_group)
  1792. load = source_load(i, load_idx);
  1793. else
  1794. load = target_load(i, load_idx);
  1795. avg_load += load;
  1796. }
  1797. /* Adjust by relative CPU power of the group */
  1798. avg_load = sg_div_cpu_power(group,
  1799. avg_load * SCHED_LOAD_SCALE);
  1800. if (local_group) {
  1801. this_load = avg_load;
  1802. this = group;
  1803. } else if (avg_load < min_load) {
  1804. min_load = avg_load;
  1805. idlest = group;
  1806. }
  1807. } while (group = group->next, group != sd->groups);
  1808. if (!idlest || 100*this_load < imbalance*min_load)
  1809. return NULL;
  1810. return idlest;
  1811. }
  1812. /*
  1813. * find_idlest_cpu - find the idlest cpu among the cpus in group.
  1814. */
  1815. static int
  1816. find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
  1817. {
  1818. unsigned long load, min_load = ULONG_MAX;
  1819. int idlest = -1;
  1820. int i;
  1821. /* Traverse only the allowed CPUs */
  1822. for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
  1823. load = weighted_cpuload(i);
  1824. if (load < min_load || (load == min_load && i == this_cpu)) {
  1825. min_load = load;
  1826. idlest = i;
  1827. }
  1828. }
  1829. return idlest;
  1830. }
  1831. /*
  1832. * sched_balance_self: balance the current task (running on cpu) in domains
  1833. * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
  1834. * SD_BALANCE_EXEC.
  1835. *
  1836. * Balance, ie. select the least loaded group.
  1837. *
  1838. * Returns the target CPU number, or the same CPU if no balancing is needed.
  1839. *
  1840. * preempt must be disabled.
  1841. */
  1842. static int sched_balance_self(int cpu, int flag)
  1843. {
  1844. struct task_struct *t = current;
  1845. struct sched_domain *tmp, *sd = NULL;
  1846. for_each_domain(cpu, tmp) {
  1847. /*
  1848. * If power savings logic is enabled for a domain, stop there.
  1849. */
  1850. if (tmp->flags & SD_POWERSAVINGS_BALANCE)
  1851. break;
  1852. if (tmp->flags & flag)
  1853. sd = tmp;
  1854. }
  1855. if (sd)
  1856. update_shares(sd);
  1857. while (sd) {
  1858. struct sched_group *group;
  1859. int new_cpu, weight;
  1860. if (!(sd->flags & flag)) {
  1861. sd = sd->child;
  1862. continue;
  1863. }
  1864. group = find_idlest_group(sd, t, cpu);
  1865. if (!group) {
  1866. sd = sd->child;
  1867. continue;
  1868. }
  1869. new_cpu = find_idlest_cpu(group, t, cpu);
  1870. if (new_cpu == -1 || new_cpu == cpu) {
  1871. /* Now try balancing at a lower domain level of cpu */
  1872. sd = sd->child;
  1873. continue;
  1874. }
  1875. /* Now try balancing at a lower domain level of new_cpu */
  1876. cpu = new_cpu;
  1877. weight = cpumask_weight(sched_domain_span(sd));
  1878. sd = NULL;
  1879. for_each_domain(cpu, tmp) {
  1880. if (weight <= cpumask_weight(sched_domain_span(tmp)))
  1881. break;
  1882. if (tmp->flags & flag)
  1883. sd = tmp;
  1884. }
  1885. /* while loop will break here if sd == NULL */
  1886. }
  1887. return cpu;
  1888. }
  1889. #endif /* CONFIG_SMP */
  1890. /***
  1891. * try_to_wake_up - wake up a thread
  1892. * @p: the to-be-woken-up thread
  1893. * @state: the mask of task states that can be woken
  1894. * @sync: do a synchronous wakeup?
  1895. *
  1896. * Put it on the run-queue if it's not already there. The "current"
  1897. * thread is always on the run-queue (except when the actual
  1898. * re-schedule is in progress), and as such you're allowed to do
  1899. * the simpler "current->state = TASK_RUNNING" to mark yourself
  1900. * runnable without the overhead of this.
  1901. *
  1902. * returns failure only if the task is already active.
  1903. */
  1904. static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
  1905. {
  1906. int cpu, orig_cpu, this_cpu, success = 0;
  1907. unsigned long flags;
  1908. long old_state;
  1909. struct rq *rq;
  1910. if (!sched_feat(SYNC_WAKEUPS))
  1911. sync = 0;
  1912. #ifdef CONFIG_SMP
  1913. if (sched_feat(LB_WAKEUP_UPDATE)) {
  1914. struct sched_domain *sd;
  1915. this_cpu = raw_smp_processor_id();
  1916. cpu = task_cpu(p);
  1917. for_each_domain(this_cpu, sd) {
  1918. if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
  1919. update_shares(sd);
  1920. break;
  1921. }
  1922. }
  1923. }
  1924. #endif
  1925. smp_wmb();
  1926. rq = task_rq_lock(p, &flags);
  1927. update_rq_clock(rq);
  1928. old_state = p->state;
  1929. if (!(old_state & state))
  1930. goto out;
  1931. if (p->se.on_rq)
  1932. goto out_running;
  1933. cpu = task_cpu(p);
  1934. orig_cpu = cpu;
  1935. this_cpu = smp_processor_id();
  1936. #ifdef CONFIG_SMP
  1937. if (unlikely(task_running(rq, p)))
  1938. goto out_activate;
  1939. cpu = p->sched_class->select_task_rq(p, sync);
  1940. if (cpu != orig_cpu) {
  1941. set_task_cpu(p, cpu);
  1942. task_rq_unlock(rq, &flags);
  1943. /* might preempt at this point */
  1944. rq = task_rq_lock(p, &flags);
  1945. old_state = p->state;
  1946. if (!(old_state & state))
  1947. goto out;
  1948. if (p->se.on_rq)
  1949. goto out_running;
  1950. this_cpu = smp_processor_id();
  1951. cpu = task_cpu(p);
  1952. }
  1953. #ifdef CONFIG_SCHEDSTATS
  1954. schedstat_inc(rq, ttwu_count);
  1955. if (cpu == this_cpu)
  1956. schedstat_inc(rq, ttwu_local);
  1957. else {
  1958. struct sched_domain *sd;
  1959. for_each_domain(this_cpu, sd) {
  1960. if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
  1961. schedstat_inc(sd, ttwu_wake_remote);
  1962. break;
  1963. }
  1964. }
  1965. }
  1966. #endif /* CONFIG_SCHEDSTATS */
  1967. out_activate:
  1968. #endif /* CONFIG_SMP */
  1969. schedstat_inc(p, se.nr_wakeups);
  1970. if (sync)
  1971. schedstat_inc(p, se.nr_wakeups_sync);
  1972. if (orig_cpu != cpu)
  1973. schedstat_inc(p, se.nr_wakeups_migrate);
  1974. if (cpu == this_cpu)
  1975. schedstat_inc(p, se.nr_wakeups_local);
  1976. else
  1977. schedstat_inc(p, se.nr_wakeups_remote);
  1978. activate_task(rq, p, 1);
  1979. success = 1;
  1980. out_running:
  1981. trace_sched_wakeup(rq, p, success);
  1982. check_preempt_curr(rq, p, sync);
  1983. p->state = TASK_RUNNING;
  1984. #ifdef CONFIG_SMP
  1985. if (p->sched_class->task_wake_up)
  1986. p->sched_class->task_wake_up(rq, p);
  1987. #endif
  1988. out:
  1989. current->se.last_wakeup = current->se.sum_exec_runtime;
  1990. task_rq_unlock(rq, &flags);
  1991. return success;
  1992. }
  1993. int wake_up_process(struct task_struct *p)
  1994. {
  1995. return try_to_wake_up(p, TASK_ALL, 0);
  1996. }
  1997. EXPORT_SYMBOL(wake_up_process);
  1998. int wake_up_state(struct task_struct *p, unsigned int state)
  1999. {
  2000. return try_to_wake_up(p, state, 0);
  2001. }
  2002. /*
  2003. * Perform scheduler related setup for a newly forked process p.
  2004. * p is forked by current.
  2005. *
  2006. * __sched_fork() is basic setup used by init_idle() too:
  2007. */
  2008. static void __sched_fork(struct task_struct *p)
  2009. {
  2010. p->se.exec_start = 0;
  2011. p->se.sum_exec_runtime = 0;
  2012. p->se.prev_sum_exec_runtime = 0;
  2013. p->se.last_wakeup = 0;
  2014. p->se.avg_overlap = 0;
  2015. #ifdef CONFIG_SCHEDSTATS
  2016. p->se.wait_start = 0;
  2017. p->se.sum_sleep_runtime = 0;
  2018. p->se.sleep_start = 0;
  2019. p->se.block_start = 0;
  2020. p->se.sleep_max = 0;
  2021. p->se.block_max = 0;
  2022. p->se.exec_max = 0;
  2023. p->se.slice_max = 0;
  2024. p->se.wait_max = 0;
  2025. #endif
  2026. INIT_LIST_HEAD(&p->rt.run_list);
  2027. p->se.on_rq = 0;
  2028. INIT_LIST_HEAD(&p->se.group_node);
  2029. #ifdef CONFIG_PREEMPT_NOTIFIERS
  2030. INIT_HLIST_HEAD(&p->preempt_notifiers);
  2031. #endif
  2032. /*
  2033. * We mark the process as running here, but have not actually
  2034. * inserted it onto the runqueue yet. This guarantees that
  2035. * nobody will actually run it, and a signal or other external
  2036. * event cannot wake it up and insert it on the runqueue either.
  2037. */
  2038. p->state = TASK_RUNNING;
  2039. }
  2040. /*
  2041. * fork()/clone()-time setup:
  2042. */
  2043. void sched_fork(struct task_struct *p, int clone_flags)
  2044. {
  2045. int cpu = get_cpu();
  2046. __sched_fork(p);
  2047. #ifdef CONFIG_SMP
  2048. cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
  2049. #endif
  2050. set_task_cpu(p, cpu);
  2051. /*
  2052. * Make sure we do not leak PI boosting priority to the child:
  2053. */
  2054. p->prio = current->normal_prio;
  2055. if (!rt_prio(p->prio))
  2056. p->sched_class = &fair_sched_class;
  2057. #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
  2058. if (likely(sched_info_on()))
  2059. memset(&p->sched_info, 0, sizeof(p->sched_info));
  2060. #endif
  2061. #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
  2062. p->oncpu = 0;
  2063. #endif
  2064. #ifdef CONFIG_PREEMPT
  2065. /* Want to start with kernel preemption disabled. */
  2066. task_thread_info(p)->preempt_count = 1;
  2067. #endif
  2068. put_cpu();
  2069. }
  2070. /*
  2071. * wake_up_new_task - wake up a newly created task for the first time.
  2072. *
  2073. * This function will do some initial scheduler statistics housekeeping
  2074. * that must be done for every newly created context, then puts the task
  2075. * on the runqueue and wakes it.
  2076. */
  2077. void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
  2078. {
  2079. unsigned long flags;
  2080. struct rq *rq;
  2081. rq = task_rq_lock(p, &flags);
  2082. BUG_ON(p->state != TASK_RUNNING);
  2083. update_rq_clock(rq);
  2084. p->prio = effective_prio(p);
  2085. if (!p->sched_class->task_new || !current->se.on_rq) {
  2086. activate_task(rq, p, 0);
  2087. } else {
  2088. /*
  2089. * Let the scheduling class do new task startup
  2090. * management (if any):
  2091. */
  2092. p->sched_class->task_new(rq, p);
  2093. inc_nr_running(rq);
  2094. }
  2095. trace_sched_wakeup_new(rq, p, 1);
  2096. check_preempt_curr(rq, p, 0);
  2097. #ifdef CONFIG_SMP
  2098. if (p->sched_class->task_wake_up)
  2099. p->sched_class->task_wake_up(rq, p);
  2100. #endif
  2101. task_rq_unlock(rq, &flags);
  2102. }
  2103. #ifdef CONFIG_PREEMPT_NOTIFIERS
  2104. /**
  2105. * preempt_notifier_register - tell me when current is being being preempted & rescheduled
  2106. * @notifier: notifier struct to register
  2107. */
  2108. void preempt_notifier_register(struct preempt_notifier *notifier)
  2109. {
  2110. hlist_add_head(&notifier->link, &current->preempt_notifiers);
  2111. }
  2112. EXPORT_SYMBOL_GPL(preempt_notifier_register);
  2113. /**
  2114. * preempt_notifier_unregister - no longer interested in preemption notifications
  2115. * @notifier: notifier struct to unregister
  2116. *
  2117. * This is safe to call from within a preemption notifier.
  2118. */
  2119. void preempt_notifier_unregister(struct preempt_notifier *notifier)
  2120. {
  2121. hlist_del(&notifier->link);
  2122. }
  2123. EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
  2124. static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
  2125. {
  2126. struct preempt_notifier *notifier;
  2127. struct hlist_node *node;
  2128. hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
  2129. notifier->ops->sched_in(notifier, raw_smp_processor_id());
  2130. }
  2131. static void
  2132. fire_sched_out_preempt_notifiers(struct task_struct *curr,
  2133. struct task_struct *next)
  2134. {
  2135. struct preempt_notifier *notifier;
  2136. struct hlist_node *node;
  2137. hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
  2138. notifier->ops->sched_out(notifier, next);
  2139. }
  2140. #else /* !CONFIG_PREEMPT_NOTIFIERS */
  2141. static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
  2142. {
  2143. }
  2144. static void
  2145. fire_sched_out_preempt_notifiers(struct task_struct *curr,
  2146. struct task_struct *next)
  2147. {
  2148. }
  2149. #endif /* CONFIG_PREEMPT_NOTIFIERS */
  2150. /**
  2151. * prepare_task_switch - prepare to switch tasks
  2152. * @rq: the runqueue preparing to switch
  2153. * @prev: the current task that is being switched out
  2154. * @next: the task we are going to switch to.
  2155. *
  2156. * This is called with the rq lock held and interrupts off. It must
  2157. * be paired with a subsequent finish_task_switch after the context
  2158. * switch.
  2159. *
  2160. * prepare_task_switch sets up locking and calls architecture specific
  2161. * hooks.
  2162. */
  2163. static inline void
  2164. prepare_task_switch(struct rq *rq, struct task_struct *prev,
  2165. struct task_struct *next)
  2166. {
  2167. fire_sched_out_preempt_notifiers(prev, next);
  2168. prepare_lock_switch(rq, next);
  2169. prepare_arch_switch(next);
  2170. }
  2171. /**
  2172. * finish_task_switch - clean up after a task-switch
  2173. * @rq: runqueue associated with task-switch
  2174. * @prev: the thread we just switched away from.
  2175. *
  2176. * finish_task_switch must be called after the context switch, paired
  2177. * with a prepare_task_switch call before the context switch.
  2178. * finish_task_switch will reconcile locking set up by prepare_task_switch,
  2179. * and do any other architecture-specific cleanup actions.
  2180. *
  2181. * Note that we may have delayed dropping an mm in context_switch(). If
  2182. * so, we finish that here outside of the runqueue lock. (Doing it
  2183. * with the lock held can cause deadlocks; see schedule() for
  2184. * details.)
  2185. */
  2186. static void finish_task_switch(struct rq *rq, struct task_struct *prev)
  2187. __releases(rq->lock)
  2188. {
  2189. struct mm_struct *mm = rq->prev_mm;
  2190. long prev_state;
  2191. rq->prev_mm = NULL;
  2192. /*
  2193. * A task struct has one reference for the use as "current".
  2194. * If a task dies, then it sets TASK_DEAD in tsk->state and calls
  2195. * schedule one last time. The schedule call will never return, and
  2196. * the scheduled task must drop that reference.
  2197. * The test for TASK_DEAD must occur while the runqueue locks are
  2198. * still held, otherwise prev could be scheduled on another cpu, die
  2199. * there before we look at prev->state, and then the reference would
  2200. * be dropped twice.
  2201. * Manfred Spraul <manfred@colorfullife.com>
  2202. */
  2203. prev_state = prev->state;
  2204. finish_arch_switch(prev);
  2205. finish_lock_switch(rq, prev);
  2206. #ifdef CONFIG_SMP
  2207. if (current->sched_class->post_schedule)
  2208. current->sched_class->post_schedule(rq);
  2209. #endif
  2210. fire_sched_in_preempt_notifiers(current);
  2211. if (mm)
  2212. mmdrop(mm);
  2213. if (unlikely(prev_state == TASK_DEAD)) {
  2214. /*
  2215. * Remove function-return probe instances associated with this
  2216. * task and put them back on the free list.
  2217. */
  2218. kprobe_flush_task(prev);
  2219. put_task_struct(prev);
  2220. }
  2221. }
  2222. /**
  2223. * schedule_tail - first thing a freshly forked thread must call.
  2224. * @prev: the thread we just switched away from.
  2225. */
  2226. asmlinkage void schedule_tail(struct task_struct *prev)
  2227. __releases(rq->lock)
  2228. {
  2229. struct rq *rq = this_rq();
  2230. finish_task_switch(rq, prev);
  2231. #ifdef __ARCH_WANT_UNLOCKED_CTXSW
  2232. /* In this case, finish_task_switch does not reenable preemption */
  2233. preempt_enable();
  2234. #endif
  2235. if (current->set_child_tid)
  2236. put_user(task_pid_vnr(current), current->set_child_tid);
  2237. }
  2238. /*
  2239. * context_switch - switch to the new MM and the new
  2240. * thread's register state.
  2241. */
  2242. static inline void
  2243. context_switch(struct rq *rq, struct task_struct *prev,
  2244. struct task_struct *next)
  2245. {
  2246. struct mm_struct *mm, *oldmm;
  2247. prepare_task_switch(rq, prev, next);
  2248. trace_sched_switch(rq, prev, next);
  2249. mm = next->mm;
  2250. oldmm = prev->active_mm;
  2251. /*
  2252. * For paravirt, this is coupled with an exit in switch_to to
  2253. * combine the page table reload and the switch backend into
  2254. * one hypercall.
  2255. */
  2256. arch_enter_lazy_cpu_mode();
  2257. if (unlikely(!mm)) {
  2258. next->active_mm = oldmm;
  2259. atomic_inc(&oldmm->mm_count);
  2260. enter_lazy_tlb(oldmm, next);
  2261. } else
  2262. switch_mm(oldmm, mm, next);
  2263. if (unlikely(!prev->mm)) {
  2264. prev->active_mm = NULL;
  2265. rq->prev_mm = oldmm;
  2266. }
  2267. /*
  2268. * Since the runqueue lock will be released by the next
  2269. * task (which is an invalid locking op but in the case
  2270. * of the scheduler it's an obvious special-case), so we
  2271. * do an early lockdep release here:
  2272. */
  2273. #ifndef __ARCH_WANT_UNLOCKED_CTXSW
  2274. spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
  2275. #endif
  2276. /* Here we just switch the register state and the stack. */
  2277. switch_to(prev, next, prev);
  2278. barrier();
  2279. /*
  2280. * this_rq must be evaluated again because prev may have moved
  2281. * CPUs since it called schedule(), thus the 'rq' on its stack
  2282. * frame will be invalid.
  2283. */
  2284. finish_task_switch(this_rq(), prev);
  2285. }
  2286. /*
  2287. * nr_running, nr_uninterruptible and nr_context_switches:
  2288. *
  2289. * externally visible scheduler statistics: current number of runnable
  2290. * threads, current number of uninterruptible-sleeping threads, total
  2291. * number of context switches performed since bootup.
  2292. */
  2293. unsigned long nr_running(void)
  2294. {
  2295. unsigned long i, sum = 0;
  2296. for_each_online_cpu(i)
  2297. sum += cpu_rq(i)->nr_running;
  2298. return sum;
  2299. }
  2300. unsigned long nr_uninterruptible(void)
  2301. {
  2302. unsigned long i, sum = 0;
  2303. for_each_possible_cpu(i)
  2304. sum += cpu_rq(i)->nr_uninterruptible;
  2305. /*
  2306. * Since we read the counters lockless, it might be slightly
  2307. * inaccurate. Do not allow it to go below zero though:
  2308. */
  2309. if (unlikely((long)sum < 0))
  2310. sum = 0;
  2311. return sum;
  2312. }
  2313. unsigned long long nr_context_switches(void)
  2314. {
  2315. int i;
  2316. unsigned long long sum = 0;
  2317. for_each_possible_cpu(i)
  2318. sum += cpu_rq(i)->nr_switches;
  2319. return sum;
  2320. }
  2321. unsigned long nr_iowait(void)
  2322. {
  2323. unsigned long i, sum = 0;
  2324. for_each_possible_cpu(i)
  2325. sum += atomic_read(&cpu_rq(i)->nr_iowait);
  2326. return sum;
  2327. }
  2328. unsigned long nr_active(void)
  2329. {
  2330. unsigned long i, running = 0, uninterruptible = 0;
  2331. for_each_online_cpu(i) {
  2332. running += cpu_rq(i)->nr_running;
  2333. uninterruptible += cpu_rq(i)->nr_uninterruptible;
  2334. }
  2335. if (unlikely((long)uninterruptible < 0))
  2336. uninterruptible = 0;
  2337. return running + uninterruptible;
  2338. }
  2339. /*
  2340. * Update rq->cpu_load[] statistics. This function is usually called every
  2341. * scheduler tick (TICK_NSEC).
  2342. */
  2343. static void update_cpu_load(struct rq *this_rq)
  2344. {
  2345. unsigned long this_load = this_rq->load.weight;
  2346. int i, scale;
  2347. this_rq->nr_load_updates++;
  2348. /* Update our load: */
  2349. for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
  2350. unsigned long old_load, new_load;
  2351. /* scale is effectively 1 << i now, and >> i divides by scale */
  2352. old_load = this_rq->cpu_load[i];
  2353. new_load = this_load;
  2354. /*
  2355. * Round up the averaging division if load is increasing. This
  2356. * prevents us from getting stuck on 9 if the load is 10, for
  2357. * example.
  2358. */
  2359. if (new_load > old_load)
  2360. new_load += scale-1;
  2361. this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
  2362. }
  2363. }
  2364. #ifdef CONFIG_SMP
  2365. /*
  2366. * double_rq_lock - safely lock two runqueues
  2367. *
  2368. * Note this does not disable interrupts like task_rq_lock,
  2369. * you need to do so manually before calling.
  2370. */
  2371. static void double_rq_lock(struct rq *rq1, struct rq *rq2)
  2372. __acquires(rq1->lock)
  2373. __acquires(rq2->lock)
  2374. {
  2375. BUG_ON(!irqs_disabled());
  2376. if (rq1 == rq2) {
  2377. spin_lock(&rq1->lock);
  2378. __acquire(rq2->lock); /* Fake it out ;) */
  2379. } else {
  2380. if (rq1 < rq2) {
  2381. spin_lock(&rq1->lock);
  2382. spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
  2383. } else {
  2384. spin_lock(&rq2->lock);
  2385. spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
  2386. }
  2387. }
  2388. update_rq_clock(rq1);
  2389. update_rq_clock(rq2);
  2390. }
  2391. /*
  2392. * double_rq_unlock - safely unlock two runqueues
  2393. *
  2394. * Note this does not restore interrupts like task_rq_unlock,
  2395. * you need to do so manually after calling.
  2396. */
  2397. static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
  2398. __releases(rq1->lock)
  2399. __releases(rq2->lock)
  2400. {
  2401. spin_unlock(&rq1->lock);
  2402. if (rq1 != rq2)
  2403. spin_unlock(&rq2->lock);
  2404. else
  2405. __release(rq2->lock);
  2406. }
  2407. /*
  2408. * If dest_cpu is allowed for this process, migrate the task to it.
  2409. * This is accomplished by forcing the cpu_allowed mask to only
  2410. * allow dest_cpu, which will force the cpu onto dest_cpu. Then
  2411. * the cpu_allowed mask is restored.
  2412. */
  2413. static void sched_migrate_task(struct task_struct *p, int dest_cpu)
  2414. {
  2415. struct migration_req req;
  2416. unsigned long flags;
  2417. struct rq *rq;
  2418. rq = task_rq_lock(p, &flags);
  2419. if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
  2420. || unlikely(!cpu_active(dest_cpu)))
  2421. goto out;
  2422. /* force the process onto the specified CPU */
  2423. if (migrate_task(p, dest_cpu, &req)) {
  2424. /* Need to wait for migration thread (might exit: take ref). */
  2425. struct task_struct *mt = rq->migration_thread;
  2426. get_task_struct(mt);
  2427. task_rq_unlock(rq, &flags);
  2428. wake_up_process(mt);
  2429. put_task_struct(mt);
  2430. wait_for_completion(&req.done);
  2431. return;
  2432. }
  2433. out:
  2434. task_rq_unlock(rq, &flags);
  2435. }
  2436. /*
  2437. * sched_exec - execve() is a valuable balancing opportunity, because at
  2438. * this point the task has the smallest effective memory and cache footprint.
  2439. */
  2440. void sched_exec(void)
  2441. {
  2442. int new_cpu, this_cpu = get_cpu();
  2443. new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
  2444. put_cpu();
  2445. if (new_cpu != this_cpu)
  2446. sched_migrate_task(current, new_cpu);
  2447. }
  2448. /*
  2449. * pull_task - move a task from a remote runqueue to the local runqueue.
  2450. * Both runqueues must be locked.
  2451. */
  2452. static void pull_task(struct rq *src_rq, struct task_struct *p,
  2453. struct rq *this_rq, int this_cpu)
  2454. {
  2455. deactivate_task(src_rq, p, 0);
  2456. set_task_cpu(p, this_cpu);
  2457. activate_task(this_rq, p, 0);
  2458. /*
  2459. * Note that idle threads have a prio of MAX_PRIO, for this test
  2460. * to be always true for them.
  2461. */
  2462. check_preempt_curr(this_rq, p, 0);
  2463. }
  2464. /*
  2465. * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
  2466. */
  2467. static
  2468. int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
  2469. struct sched_domain *sd, enum cpu_idle_type idle,
  2470. int *all_pinned)
  2471. {
  2472. /*
  2473. * We do not migrate tasks that are:
  2474. * 1) running (obviously), or
  2475. * 2) cannot be migrated to this CPU due to cpus_allowed, or
  2476. * 3) are cache-hot on their current CPU.
  2477. */
  2478. if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
  2479. schedstat_inc(p, se.nr_failed_migrations_affine);
  2480. return 0;
  2481. }
  2482. *all_pinned = 0;
  2483. if (task_running(rq, p)) {
  2484. schedstat_inc(p, se.nr_failed_migrations_running);
  2485. return 0;
  2486. }
  2487. /*
  2488. * Aggressive migration if:
  2489. * 1) task is cache cold, or
  2490. * 2) too many balance attempts have failed.
  2491. */
  2492. if (!task_hot(p, rq->clock, sd) ||
  2493. sd->nr_balance_failed > sd->cache_nice_tries) {
  2494. #ifdef CONFIG_SCHEDSTATS
  2495. if (task_hot(p, rq->clock, sd)) {
  2496. schedstat_inc(sd, lb_hot_gained[idle]);
  2497. schedstat_inc(p, se.nr_forced_migrations);
  2498. }
  2499. #endif
  2500. return 1;
  2501. }
  2502. if (task_hot(p, rq->clock, sd)) {
  2503. schedstat_inc(p, se.nr_failed_migrations_hot);
  2504. return 0;
  2505. }
  2506. return 1;
  2507. }
  2508. static unsigned long
  2509. balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
  2510. unsigned long max_load_move, struct sched_domain *sd,
  2511. enum cpu_idle_type idle, int *all_pinned,
  2512. int *this_best_prio, struct rq_iterator *iterator)
  2513. {
  2514. int loops = 0, pulled = 0, pinned = 0;
  2515. struct task_struct *p;
  2516. long rem_load_move = max_load_move;
  2517. if (max_load_move == 0)
  2518. goto out;
  2519. pinned = 1;
  2520. /*
  2521. * Start the load-balancing iterator:
  2522. */
  2523. p = iterator->start(iterator->arg);
  2524. next:
  2525. if (!p || loops++ > sysctl_sched_nr_migrate)
  2526. goto out;
  2527. if ((p->se.load.weight >> 1) > rem_load_move ||
  2528. !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
  2529. p = iterator->next(iterator->arg);
  2530. goto next;
  2531. }
  2532. pull_task(busiest, p, this_rq, this_cpu);
  2533. pulled++;
  2534. rem_load_move -= p->se.load.weight;
  2535. /*
  2536. * We only want to steal up to the prescribed amount of weighted load.
  2537. */
  2538. if (rem_load_move > 0) {
  2539. if (p->prio < *this_best_prio)
  2540. *this_best_prio = p->prio;
  2541. p = iterator->next(iterator->arg);
  2542. goto next;
  2543. }
  2544. out:
  2545. /*
  2546. * Right now, this is one of only two places pull_task() is called,
  2547. * so we can safely collect pull_task() stats here rather than
  2548. * inside pull_task().
  2549. */
  2550. schedstat_add(sd, lb_gained[idle], pulled);
  2551. if (all_pinned)
  2552. *all_pinned = pinned;
  2553. return max_load_move - rem_load_move;
  2554. }
  2555. /*
  2556. * move_tasks tries to move up to max_load_move weighted load from busiest to
  2557. * this_rq, as part of a balancing operation within domain "sd".
  2558. * Returns 1 if successful and 0 otherwise.
  2559. *
  2560. * Called with both runqueues locked.
  2561. */
  2562. static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
  2563. unsigned long max_load_move,
  2564. struct sched_domain *sd, enum cpu_idle_type idle,
  2565. int *all_pinned)
  2566. {
  2567. const struct sched_class *class = sched_class_highest;
  2568. unsigned long total_load_moved = 0;
  2569. int this_best_prio = this_rq->curr->prio;
  2570. do {
  2571. total_load_moved +=
  2572. class->load_balance(this_rq, this_cpu, busiest,
  2573. max_load_move - total_load_moved,
  2574. sd, idle, all_pinned, &this_best_prio);
  2575. class = class->next;
  2576. if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
  2577. break;
  2578. } while (class && max_load_move > total_load_moved);
  2579. return total_load_moved > 0;
  2580. }
  2581. static int
  2582. iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
  2583. struct sched_domain *sd, enum cpu_idle_type idle,
  2584. struct rq_iterator *iterator)
  2585. {
  2586. struct task_struct *p = iterator->start(iterator->arg);
  2587. int pinned = 0;
  2588. while (p) {
  2589. if (can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
  2590. pull_task(busiest, p, this_rq, this_cpu);
  2591. /*
  2592. * Right now, this is only the second place pull_task()
  2593. * is called, so we can safely collect pull_task()
  2594. * stats here rather than inside pull_task().
  2595. */
  2596. schedstat_inc(sd, lb_gained[idle]);
  2597. return 1;
  2598. }
  2599. p = iterator->next(iterator->arg);
  2600. }
  2601. return 0;
  2602. }
  2603. /*
  2604. * move_one_task tries to move exactly one task from busiest to this_rq, as
  2605. * part of active balancing operations within "domain".
  2606. * Returns 1 if successful and 0 otherwise.
  2607. *
  2608. * Called with both runqueues locked.
  2609. */
  2610. static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
  2611. struct sched_domain *sd, enum cpu_idle_type idle)
  2612. {
  2613. const struct sched_class *class;
  2614. for (class = sched_class_highest; class; class = class->next)
  2615. if (class->move_one_task(this_rq, this_cpu, busiest, sd, idle))
  2616. return 1;
  2617. return 0;
  2618. }
  2619. /*
  2620. * find_busiest_group finds and returns the busiest CPU group within the
  2621. * domain. It calculates and returns the amount of weighted load which
  2622. * should be moved to restore balance via the imbalance parameter.
  2623. */
  2624. static struct sched_group *
  2625. find_busiest_group(struct sched_domain *sd, int this_cpu,
  2626. unsigned long *imbalance, enum cpu_idle_type idle,
  2627. int *sd_idle, const struct cpumask *cpus, int *balance)
  2628. {
  2629. struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
  2630. unsigned long max_load, avg_load, total_load, this_load, total_pwr;
  2631. unsigned long max_pull;
  2632. unsigned long busiest_load_per_task, busiest_nr_running;
  2633. unsigned long this_load_per_task, this_nr_running;
  2634. int load_idx, group_imb = 0;
  2635. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  2636. int power_savings_balance = 1;
  2637. unsigned long leader_nr_running = 0, min_load_per_task = 0;
  2638. unsigned long min_nr_running = ULONG_MAX;
  2639. struct sched_group *group_min = NULL, *group_leader = NULL;
  2640. #endif
  2641. max_load = this_load = total_load = total_pwr = 0;
  2642. busiest_load_per_task = busiest_nr_running = 0;
  2643. this_load_per_task = this_nr_running = 0;
  2644. if (idle == CPU_NOT_IDLE)
  2645. load_idx = sd->busy_idx;
  2646. else if (idle == CPU_NEWLY_IDLE)
  2647. load_idx = sd->newidle_idx;
  2648. else
  2649. load_idx = sd->idle_idx;
  2650. do {
  2651. unsigned long load, group_capacity, max_cpu_load, min_cpu_load;
  2652. int local_group;
  2653. int i;
  2654. int __group_imb = 0;
  2655. unsigned int balance_cpu = -1, first_idle_cpu = 0;
  2656. unsigned long sum_nr_running, sum_weighted_load;
  2657. unsigned long sum_avg_load_per_task;
  2658. unsigned long avg_load_per_task;
  2659. local_group = cpumask_test_cpu(this_cpu,
  2660. sched_group_cpus(group));
  2661. if (local_group)
  2662. balance_cpu = cpumask_first(sched_group_cpus(group));
  2663. /* Tally up the load of all CPUs in the group */
  2664. sum_weighted_load = sum_nr_running = avg_load = 0;
  2665. sum_avg_load_per_task = avg_load_per_task = 0;
  2666. max_cpu_load = 0;
  2667. min_cpu_load = ~0UL;
  2668. for_each_cpu_and(i, sched_group_cpus(group), cpus) {
  2669. struct rq *rq = cpu_rq(i);
  2670. if (*sd_idle && rq->nr_running)
  2671. *sd_idle = 0;
  2672. /* Bias balancing toward cpus of our domain */
  2673. if (local_group) {
  2674. if (idle_cpu(i) && !first_idle_cpu) {
  2675. first_idle_cpu = 1;
  2676. balance_cpu = i;
  2677. }
  2678. load = target_load(i, load_idx);
  2679. } else {
  2680. load = source_load(i, load_idx);
  2681. if (load > max_cpu_load)
  2682. max_cpu_load = load;
  2683. if (min_cpu_load > load)
  2684. min_cpu_load = load;
  2685. }
  2686. avg_load += load;
  2687. sum_nr_running += rq->nr_running;
  2688. sum_weighted_load += weighted_cpuload(i);
  2689. sum_avg_load_per_task += cpu_avg_load_per_task(i);
  2690. }
  2691. /*
  2692. * First idle cpu or the first cpu(busiest) in this sched group
  2693. * is eligible for doing load balancing at this and above
  2694. * domains. In the newly idle case, we will allow all the cpu's
  2695. * to do the newly idle load balance.
  2696. */
  2697. if (idle != CPU_NEWLY_IDLE && local_group &&
  2698. balance_cpu != this_cpu && balance) {
  2699. *balance = 0;
  2700. goto ret;
  2701. }
  2702. total_load += avg_load;
  2703. total_pwr += group->__cpu_power;
  2704. /* Adjust by relative CPU power of the group */
  2705. avg_load = sg_div_cpu_power(group,
  2706. avg_load * SCHED_LOAD_SCALE);
  2707. /*
  2708. * Consider the group unbalanced when the imbalance is larger
  2709. * than the average weight of two tasks.
  2710. *
  2711. * APZ: with cgroup the avg task weight can vary wildly and
  2712. * might not be a suitable number - should we keep a
  2713. * normalized nr_running number somewhere that negates
  2714. * the hierarchy?
  2715. */
  2716. avg_load_per_task = sg_div_cpu_power(group,
  2717. sum_avg_load_per_task * SCHED_LOAD_SCALE);
  2718. if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
  2719. __group_imb = 1;
  2720. group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
  2721. if (local_group) {
  2722. this_load = avg_load;
  2723. this = group;
  2724. this_nr_running = sum_nr_running;
  2725. this_load_per_task = sum_weighted_load;
  2726. } else if (avg_load > max_load &&
  2727. (sum_nr_running > group_capacity || __group_imb)) {
  2728. max_load = avg_load;
  2729. busiest = group;
  2730. busiest_nr_running = sum_nr_running;
  2731. busiest_load_per_task = sum_weighted_load;
  2732. group_imb = __group_imb;
  2733. }
  2734. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  2735. /*
  2736. * Busy processors will not participate in power savings
  2737. * balance.
  2738. */
  2739. if (idle == CPU_NOT_IDLE ||
  2740. !(sd->flags & SD_POWERSAVINGS_BALANCE))
  2741. goto group_next;
  2742. /*
  2743. * If the local group is idle or completely loaded
  2744. * no need to do power savings balance at this domain
  2745. */
  2746. if (local_group && (this_nr_running >= group_capacity ||
  2747. !this_nr_running))
  2748. power_savings_balance = 0;
  2749. /*
  2750. * If a group is already running at full capacity or idle,
  2751. * don't include that group in power savings calculations
  2752. */
  2753. if (!power_savings_balance || sum_nr_running >= group_capacity
  2754. || !sum_nr_running)
  2755. goto group_next;
  2756. /*
  2757. * Calculate the group which has the least non-idle load.
  2758. * This is the group from where we need to pick up the load
  2759. * for saving power
  2760. */
  2761. if ((sum_nr_running < min_nr_running) ||
  2762. (sum_nr_running == min_nr_running &&
  2763. cpumask_first(sched_group_cpus(group)) >
  2764. cpumask_first(sched_group_cpus(group_min)))) {
  2765. group_min = group;
  2766. min_nr_running = sum_nr_running;
  2767. min_load_per_task = sum_weighted_load /
  2768. sum_nr_running;
  2769. }
  2770. /*
  2771. * Calculate the group which is almost near its
  2772. * capacity but still has some space to pick up some load
  2773. * from other group and save more power
  2774. */
  2775. if (sum_nr_running <= group_capacity - 1) {
  2776. if (sum_nr_running > leader_nr_running ||
  2777. (sum_nr_running == leader_nr_running &&
  2778. cpumask_first(sched_group_cpus(group)) <
  2779. cpumask_first(sched_group_cpus(group_leader)))) {
  2780. group_leader = group;
  2781. leader_nr_running = sum_nr_running;
  2782. }
  2783. }
  2784. group_next:
  2785. #endif
  2786. group = group->next;
  2787. } while (group != sd->groups);
  2788. if (!busiest || this_load >= max_load || busiest_nr_running == 0)
  2789. goto out_balanced;
  2790. avg_load = (SCHED_LOAD_SCALE * total_load) / total_pwr;
  2791. if (this_load >= avg_load ||
  2792. 100*max_load <= sd->imbalance_pct*this_load)
  2793. goto out_balanced;
  2794. busiest_load_per_task /= busiest_nr_running;
  2795. if (group_imb)
  2796. busiest_load_per_task = min(busiest_load_per_task, avg_load);
  2797. /*
  2798. * We're trying to get all the cpus to the average_load, so we don't
  2799. * want to push ourselves above the average load, nor do we wish to
  2800. * reduce the max loaded cpu below the average load, as either of these
  2801. * actions would just result in more rebalancing later, and ping-pong
  2802. * tasks around. Thus we look for the minimum possible imbalance.
  2803. * Negative imbalances (*we* are more loaded than anyone else) will
  2804. * be counted as no imbalance for these purposes -- we can't fix that
  2805. * by pulling tasks to us. Be careful of negative numbers as they'll
  2806. * appear as very large values with unsigned longs.
  2807. */
  2808. if (max_load <= busiest_load_per_task)
  2809. goto out_balanced;
  2810. /*
  2811. * In the presence of smp nice balancing, certain scenarios can have
  2812. * max load less than avg load(as we skip the groups at or below
  2813. * its cpu_power, while calculating max_load..)
  2814. */
  2815. if (max_load < avg_load) {
  2816. *imbalance = 0;
  2817. goto small_imbalance;
  2818. }
  2819. /* Don't want to pull so many tasks that a group would go idle */
  2820. max_pull = min(max_load - avg_load, max_load - busiest_load_per_task);
  2821. /* How much load to actually move to equalise the imbalance */
  2822. *imbalance = min(max_pull * busiest->__cpu_power,
  2823. (avg_load - this_load) * this->__cpu_power)
  2824. / SCHED_LOAD_SCALE;
  2825. /*
  2826. * if *imbalance is less than the average load per runnable task
  2827. * there is no gaurantee that any tasks will be moved so we'll have
  2828. * a think about bumping its value to force at least one task to be
  2829. * moved
  2830. */
  2831. if (*imbalance < busiest_load_per_task) {
  2832. unsigned long tmp, pwr_now, pwr_move;
  2833. unsigned int imbn;
  2834. small_imbalance:
  2835. pwr_move = pwr_now = 0;
  2836. imbn = 2;
  2837. if (this_nr_running) {
  2838. this_load_per_task /= this_nr_running;
  2839. if (busiest_load_per_task > this_load_per_task)
  2840. imbn = 1;
  2841. } else
  2842. this_load_per_task = cpu_avg_load_per_task(this_cpu);
  2843. if (max_load - this_load + busiest_load_per_task >=
  2844. busiest_load_per_task * imbn) {
  2845. *imbalance = busiest_load_per_task;
  2846. return busiest;
  2847. }
  2848. /*
  2849. * OK, we don't have enough imbalance to justify moving tasks,
  2850. * however we may be able to increase total CPU power used by
  2851. * moving them.
  2852. */
  2853. pwr_now += busiest->__cpu_power *
  2854. min(busiest_load_per_task, max_load);
  2855. pwr_now += this->__cpu_power *
  2856. min(this_load_per_task, this_load);
  2857. pwr_now /= SCHED_LOAD_SCALE;
  2858. /* Amount of load we'd subtract */
  2859. tmp = sg_div_cpu_power(busiest,
  2860. busiest_load_per_task * SCHED_LOAD_SCALE);
  2861. if (max_load > tmp)
  2862. pwr_move += busiest->__cpu_power *
  2863. min(busiest_load_per_task, max_load - tmp);
  2864. /* Amount of load we'd add */
  2865. if (max_load * busiest->__cpu_power <
  2866. busiest_load_per_task * SCHED_LOAD_SCALE)
  2867. tmp = sg_div_cpu_power(this,
  2868. max_load * busiest->__cpu_power);
  2869. else
  2870. tmp = sg_div_cpu_power(this,
  2871. busiest_load_per_task * SCHED_LOAD_SCALE);
  2872. pwr_move += this->__cpu_power *
  2873. min(this_load_per_task, this_load + tmp);
  2874. pwr_move /= SCHED_LOAD_SCALE;
  2875. /* Move if we gain throughput */
  2876. if (pwr_move > pwr_now)
  2877. *imbalance = busiest_load_per_task;
  2878. }
  2879. return busiest;
  2880. out_balanced:
  2881. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  2882. if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
  2883. goto ret;
  2884. if (this == group_leader && group_leader != group_min) {
  2885. *imbalance = min_load_per_task;
  2886. if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
  2887. cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
  2888. cpumask_first(sched_group_cpus(group_leader));
  2889. }
  2890. return group_min;
  2891. }
  2892. #endif
  2893. ret:
  2894. *imbalance = 0;
  2895. return NULL;
  2896. }
  2897. /*
  2898. * find_busiest_queue - find the busiest runqueue among the cpus in group.
  2899. */
  2900. static struct rq *
  2901. find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
  2902. unsigned long imbalance, const struct cpumask *cpus)
  2903. {
  2904. struct rq *busiest = NULL, *rq;
  2905. unsigned long max_load = 0;
  2906. int i;
  2907. for_each_cpu(i, sched_group_cpus(group)) {
  2908. unsigned long wl;
  2909. if (!cpumask_test_cpu(i, cpus))
  2910. continue;
  2911. rq = cpu_rq(i);
  2912. wl = weighted_cpuload(i);
  2913. if (rq->nr_running == 1 && wl > imbalance)
  2914. continue;
  2915. if (wl > max_load) {
  2916. max_load = wl;
  2917. busiest = rq;
  2918. }
  2919. }
  2920. return busiest;
  2921. }
  2922. /*
  2923. * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
  2924. * so long as it is large enough.
  2925. */
  2926. #define MAX_PINNED_INTERVAL 512
  2927. /*
  2928. * Check this_cpu to ensure it is balanced within domain. Attempt to move
  2929. * tasks if there is an imbalance.
  2930. */
  2931. static int load_balance(int this_cpu, struct rq *this_rq,
  2932. struct sched_domain *sd, enum cpu_idle_type idle,
  2933. int *balance, struct cpumask *cpus)
  2934. {
  2935. int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
  2936. struct sched_group *group;
  2937. unsigned long imbalance;
  2938. struct rq *busiest;
  2939. unsigned long flags;
  2940. cpumask_setall(cpus);
  2941. /*
  2942. * When power savings policy is enabled for the parent domain, idle
  2943. * sibling can pick up load irrespective of busy siblings. In this case,
  2944. * let the state of idle sibling percolate up as CPU_IDLE, instead of
  2945. * portraying it as CPU_NOT_IDLE.
  2946. */
  2947. if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
  2948. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  2949. sd_idle = 1;
  2950. schedstat_inc(sd, lb_count[idle]);
  2951. redo:
  2952. update_shares(sd);
  2953. group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
  2954. cpus, balance);
  2955. if (*balance == 0)
  2956. goto out_balanced;
  2957. if (!group) {
  2958. schedstat_inc(sd, lb_nobusyg[idle]);
  2959. goto out_balanced;
  2960. }
  2961. busiest = find_busiest_queue(group, idle, imbalance, cpus);
  2962. if (!busiest) {
  2963. schedstat_inc(sd, lb_nobusyq[idle]);
  2964. goto out_balanced;
  2965. }
  2966. BUG_ON(busiest == this_rq);
  2967. schedstat_add(sd, lb_imbalance[idle], imbalance);
  2968. ld_moved = 0;
  2969. if (busiest->nr_running > 1) {
  2970. /*
  2971. * Attempt to move tasks. If find_busiest_group has found
  2972. * an imbalance but busiest->nr_running <= 1, the group is
  2973. * still unbalanced. ld_moved simply stays zero, so it is
  2974. * correctly treated as an imbalance.
  2975. */
  2976. local_irq_save(flags);
  2977. double_rq_lock(this_rq, busiest);
  2978. ld_moved = move_tasks(this_rq, this_cpu, busiest,
  2979. imbalance, sd, idle, &all_pinned);
  2980. double_rq_unlock(this_rq, busiest);
  2981. local_irq_restore(flags);
  2982. /*
  2983. * some other cpu did the load balance for us.
  2984. */
  2985. if (ld_moved && this_cpu != smp_processor_id())
  2986. resched_cpu(this_cpu);
  2987. /* All tasks on this runqueue were pinned by CPU affinity */
  2988. if (unlikely(all_pinned)) {
  2989. cpumask_clear_cpu(cpu_of(busiest), cpus);
  2990. if (!cpumask_empty(cpus))
  2991. goto redo;
  2992. goto out_balanced;
  2993. }
  2994. }
  2995. if (!ld_moved) {
  2996. schedstat_inc(sd, lb_failed[idle]);
  2997. sd->nr_balance_failed++;
  2998. if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {
  2999. spin_lock_irqsave(&busiest->lock, flags);
  3000. /* don't kick the migration_thread, if the curr
  3001. * task on busiest cpu can't be moved to this_cpu
  3002. */
  3003. if (!cpumask_test_cpu(this_cpu,
  3004. &busiest->curr->cpus_allowed)) {
  3005. spin_unlock_irqrestore(&busiest->lock, flags);
  3006. all_pinned = 1;
  3007. goto out_one_pinned;
  3008. }
  3009. if (!busiest->active_balance) {
  3010. busiest->active_balance = 1;
  3011. busiest->push_cpu = this_cpu;
  3012. active_balance = 1;
  3013. }
  3014. spin_unlock_irqrestore(&busiest->lock, flags);
  3015. if (active_balance)
  3016. wake_up_process(busiest->migration_thread);
  3017. /*
  3018. * We've kicked active balancing, reset the failure
  3019. * counter.
  3020. */
  3021. sd->nr_balance_failed = sd->cache_nice_tries+1;
  3022. }
  3023. } else
  3024. sd->nr_balance_failed = 0;
  3025. if (likely(!active_balance)) {
  3026. /* We were unbalanced, so reset the balancing interval */
  3027. sd->balance_interval = sd->min_interval;
  3028. } else {
  3029. /*
  3030. * If we've begun active balancing, start to back off. This
  3031. * case may not be covered by the all_pinned logic if there
  3032. * is only 1 task on the busy runqueue (because we don't call
  3033. * move_tasks).
  3034. */
  3035. if (sd->balance_interval < sd->max_interval)
  3036. sd->balance_interval *= 2;
  3037. }
  3038. if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
  3039. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3040. ld_moved = -1;
  3041. goto out;
  3042. out_balanced:
  3043. schedstat_inc(sd, lb_balanced[idle]);
  3044. sd->nr_balance_failed = 0;
  3045. out_one_pinned:
  3046. /* tune up the balancing interval */
  3047. if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
  3048. (sd->balance_interval < sd->max_interval))
  3049. sd->balance_interval *= 2;
  3050. if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
  3051. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3052. ld_moved = -1;
  3053. else
  3054. ld_moved = 0;
  3055. out:
  3056. if (ld_moved)
  3057. update_shares(sd);
  3058. return ld_moved;
  3059. }
  3060. /*
  3061. * Check this_cpu to ensure it is balanced within domain. Attempt to move
  3062. * tasks if there is an imbalance.
  3063. *
  3064. * Called from schedule when this_rq is about to become idle (CPU_NEWLY_IDLE).
  3065. * this_rq is locked.
  3066. */
  3067. static int
  3068. load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
  3069. struct cpumask *cpus)
  3070. {
  3071. struct sched_group *group;
  3072. struct rq *busiest = NULL;
  3073. unsigned long imbalance;
  3074. int ld_moved = 0;
  3075. int sd_idle = 0;
  3076. int all_pinned = 0;
  3077. cpumask_setall(cpus);
  3078. /*
  3079. * When power savings policy is enabled for the parent domain, idle
  3080. * sibling can pick up load irrespective of busy siblings. In this case,
  3081. * let the state of idle sibling percolate up as IDLE, instead of
  3082. * portraying it as CPU_NOT_IDLE.
  3083. */
  3084. if (sd->flags & SD_SHARE_CPUPOWER &&
  3085. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3086. sd_idle = 1;
  3087. schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
  3088. redo:
  3089. update_shares_locked(this_rq, sd);
  3090. group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
  3091. &sd_idle, cpus, NULL);
  3092. if (!group) {
  3093. schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
  3094. goto out_balanced;
  3095. }
  3096. busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance, cpus);
  3097. if (!busiest) {
  3098. schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
  3099. goto out_balanced;
  3100. }
  3101. BUG_ON(busiest == this_rq);
  3102. schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
  3103. ld_moved = 0;
  3104. if (busiest->nr_running > 1) {
  3105. /* Attempt to move tasks */
  3106. double_lock_balance(this_rq, busiest);
  3107. /* this_rq->clock is already updated */
  3108. update_rq_clock(busiest);
  3109. ld_moved = move_tasks(this_rq, this_cpu, busiest,
  3110. imbalance, sd, CPU_NEWLY_IDLE,
  3111. &all_pinned);
  3112. double_unlock_balance(this_rq, busiest);
  3113. if (unlikely(all_pinned)) {
  3114. cpumask_clear_cpu(cpu_of(busiest), cpus);
  3115. if (!cpumask_empty(cpus))
  3116. goto redo;
  3117. }
  3118. }
  3119. if (!ld_moved) {
  3120. int active_balance = 0;
  3121. schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
  3122. if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
  3123. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3124. return -1;
  3125. if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
  3126. return -1;
  3127. if (sd->nr_balance_failed++ < 2)
  3128. return -1;
  3129. /*
  3130. * The only task running in a non-idle cpu can be moved to this
  3131. * cpu in an attempt to completely freeup the other CPU
  3132. * package. The same method used to move task in load_balance()
  3133. * have been extended for load_balance_newidle() to speedup
  3134. * consolidation at sched_mc=POWERSAVINGS_BALANCE_WAKEUP (2)
  3135. *
  3136. * The package power saving logic comes from
  3137. * find_busiest_group(). If there are no imbalance, then
  3138. * f_b_g() will return NULL. However when sched_mc={1,2} then
  3139. * f_b_g() will select a group from which a running task may be
  3140. * pulled to this cpu in order to make the other package idle.
  3141. * If there is no opportunity to make a package idle and if
  3142. * there are no imbalance, then f_b_g() will return NULL and no
  3143. * action will be taken in load_balance_newidle().
  3144. *
  3145. * Under normal task pull operation due to imbalance, there
  3146. * will be more than one task in the source run queue and
  3147. * move_tasks() will succeed. ld_moved will be true and this
  3148. * active balance code will not be triggered.
  3149. */
  3150. /* Lock busiest in correct order while this_rq is held */
  3151. double_lock_balance(this_rq, busiest);
  3152. /*
  3153. * don't kick the migration_thread, if the curr
  3154. * task on busiest cpu can't be moved to this_cpu
  3155. */
  3156. if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
  3157. double_unlock_balance(this_rq, busiest);
  3158. all_pinned = 1;
  3159. return ld_moved;
  3160. }
  3161. if (!busiest->active_balance) {
  3162. busiest->active_balance = 1;
  3163. busiest->push_cpu = this_cpu;
  3164. active_balance = 1;
  3165. }
  3166. double_unlock_balance(this_rq, busiest);
  3167. if (active_balance)
  3168. wake_up_process(busiest->migration_thread);
  3169. } else
  3170. sd->nr_balance_failed = 0;
  3171. update_shares_locked(this_rq, sd);
  3172. return ld_moved;
  3173. out_balanced:
  3174. schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
  3175. if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
  3176. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3177. return -1;
  3178. sd->nr_balance_failed = 0;
  3179. return 0;
  3180. }
  3181. /*
  3182. * idle_balance is called by schedule() if this_cpu is about to become
  3183. * idle. Attempts to pull tasks from other CPUs.
  3184. */
  3185. static void idle_balance(int this_cpu, struct rq *this_rq)
  3186. {
  3187. struct sched_domain *sd;
  3188. int pulled_task = 0;
  3189. unsigned long next_balance = jiffies + HZ;
  3190. cpumask_var_t tmpmask;
  3191. if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
  3192. return;
  3193. for_each_domain(this_cpu, sd) {
  3194. unsigned long interval;
  3195. if (!(sd->flags & SD_LOAD_BALANCE))
  3196. continue;
  3197. if (sd->flags & SD_BALANCE_NEWIDLE)
  3198. /* If we've pulled tasks over stop searching: */
  3199. pulled_task = load_balance_newidle(this_cpu, this_rq,
  3200. sd, tmpmask);
  3201. interval = msecs_to_jiffies(sd->balance_interval);
  3202. if (time_after(next_balance, sd->last_balance + interval))
  3203. next_balance = sd->last_balance + interval;
  3204. if (pulled_task)
  3205. break;
  3206. }
  3207. if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
  3208. /*
  3209. * We are going idle. next_balance may be set based on
  3210. * a busy processor. So reset next_balance.
  3211. */
  3212. this_rq->next_balance = next_balance;
  3213. }
  3214. free_cpumask_var(tmpmask);
  3215. }
  3216. /*
  3217. * active_load_balance is run by migration threads. It pushes running tasks
  3218. * off the busiest CPU onto idle CPUs. It requires at least 1 task to be
  3219. * running on each physical CPU where possible, and avoids physical /
  3220. * logical imbalances.
  3221. *
  3222. * Called with busiest_rq locked.
  3223. */
  3224. static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
  3225. {
  3226. int target_cpu = busiest_rq->push_cpu;
  3227. struct sched_domain *sd;
  3228. struct rq *target_rq;
  3229. /* Is there any task to move? */
  3230. if (busiest_rq->nr_running <= 1)
  3231. return;
  3232. target_rq = cpu_rq(target_cpu);
  3233. /*
  3234. * This condition is "impossible", if it occurs
  3235. * we need to fix it. Originally reported by
  3236. * Bjorn Helgaas on a 128-cpu setup.
  3237. */
  3238. BUG_ON(busiest_rq == target_rq);
  3239. /* move a task from busiest_rq to target_rq */
  3240. double_lock_balance(busiest_rq, target_rq);
  3241. update_rq_clock(busiest_rq);
  3242. update_rq_clock(target_rq);
  3243. /* Search for an sd spanning us and the target CPU. */
  3244. for_each_domain(target_cpu, sd) {
  3245. if ((sd->flags & SD_LOAD_BALANCE) &&
  3246. cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
  3247. break;
  3248. }
  3249. if (likely(sd)) {
  3250. schedstat_inc(sd, alb_count);
  3251. if (move_one_task(target_rq, target_cpu, busiest_rq,
  3252. sd, CPU_IDLE))
  3253. schedstat_inc(sd, alb_pushed);
  3254. else
  3255. schedstat_inc(sd, alb_failed);
  3256. }
  3257. double_unlock_balance(busiest_rq, target_rq);
  3258. }
  3259. #ifdef CONFIG_NO_HZ
  3260. static struct {
  3261. atomic_t load_balancer;
  3262. cpumask_var_t cpu_mask;
  3263. } nohz ____cacheline_aligned = {
  3264. .load_balancer = ATOMIC_INIT(-1),
  3265. };
  3266. /*
  3267. * This routine will try to nominate the ilb (idle load balancing)
  3268. * owner among the cpus whose ticks are stopped. ilb owner will do the idle
  3269. * load balancing on behalf of all those cpus. If all the cpus in the system
  3270. * go into this tickless mode, then there will be no ilb owner (as there is
  3271. * no need for one) and all the cpus will sleep till the next wakeup event
  3272. * arrives...
  3273. *
  3274. * For the ilb owner, tick is not stopped. And this tick will be used
  3275. * for idle load balancing. ilb owner will still be part of
  3276. * nohz.cpu_mask..
  3277. *
  3278. * While stopping the tick, this cpu will become the ilb owner if there
  3279. * is no other owner. And will be the owner till that cpu becomes busy
  3280. * or if all cpus in the system stop their ticks at which point
  3281. * there is no need for ilb owner.
  3282. *
  3283. * When the ilb owner becomes busy, it nominates another owner, during the
  3284. * next busy scheduler_tick()
  3285. */
  3286. int select_nohz_load_balancer(int stop_tick)
  3287. {
  3288. int cpu = smp_processor_id();
  3289. if (stop_tick) {
  3290. cpumask_set_cpu(cpu, nohz.cpu_mask);
  3291. cpu_rq(cpu)->in_nohz_recently = 1;
  3292. /*
  3293. * If we are going offline and still the leader, give up!
  3294. */
  3295. if (!cpu_active(cpu) &&
  3296. atomic_read(&nohz.load_balancer) == cpu) {
  3297. if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
  3298. BUG();
  3299. return 0;
  3300. }
  3301. /* time for ilb owner also to sleep */
  3302. if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
  3303. if (atomic_read(&nohz.load_balancer) == cpu)
  3304. atomic_set(&nohz.load_balancer, -1);
  3305. return 0;
  3306. }
  3307. if (atomic_read(&nohz.load_balancer) == -1) {
  3308. /* make me the ilb owner */
  3309. if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
  3310. return 1;
  3311. } else if (atomic_read(&nohz.load_balancer) == cpu)
  3312. return 1;
  3313. } else {
  3314. if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
  3315. return 0;
  3316. cpumask_clear_cpu(cpu, nohz.cpu_mask);
  3317. if (atomic_read(&nohz.load_balancer) == cpu)
  3318. if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
  3319. BUG();
  3320. }
  3321. return 0;
  3322. }
  3323. #endif
  3324. static DEFINE_SPINLOCK(balancing);
  3325. /*
  3326. * It checks each scheduling domain to see if it is due to be balanced,
  3327. * and initiates a balancing operation if so.
  3328. *
  3329. * Balancing parameters are set up in arch_init_sched_domains.
  3330. */
  3331. static void rebalance_domains(int cpu, enum cpu_idle_type idle)
  3332. {
  3333. int balance = 1;
  3334. struct rq *rq = cpu_rq(cpu);
  3335. unsigned long interval;
  3336. struct sched_domain *sd;
  3337. /* Earliest time when we have to do rebalance again */
  3338. unsigned long next_balance = jiffies + 60*HZ;
  3339. int update_next_balance = 0;
  3340. int need_serialize;
  3341. cpumask_var_t tmp;
  3342. /* Fails alloc? Rebalancing probably not a priority right now. */
  3343. if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
  3344. return;
  3345. for_each_domain(cpu, sd) {
  3346. if (!(sd->flags & SD_LOAD_BALANCE))
  3347. continue;
  3348. interval = sd->balance_interval;
  3349. if (idle != CPU_IDLE)
  3350. interval *= sd->busy_factor;
  3351. /* scale ms to jiffies */
  3352. interval = msecs_to_jiffies(interval);
  3353. if (unlikely(!interval))
  3354. interval = 1;
  3355. if (interval > HZ*NR_CPUS/10)
  3356. interval = HZ*NR_CPUS/10;
  3357. need_serialize = sd->flags & SD_SERIALIZE;
  3358. if (need_serialize) {
  3359. if (!spin_trylock(&balancing))
  3360. goto out;
  3361. }
  3362. if (time_after_eq(jiffies, sd->last_balance + interval)) {
  3363. if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
  3364. /*
  3365. * We've pulled tasks over so either we're no
  3366. * longer idle, or one of our SMT siblings is
  3367. * not idle.
  3368. */
  3369. idle = CPU_NOT_IDLE;
  3370. }
  3371. sd->last_balance = jiffies;
  3372. }
  3373. if (need_serialize)
  3374. spin_unlock(&balancing);
  3375. out:
  3376. if (time_after(next_balance, sd->last_balance + interval)) {
  3377. next_balance = sd->last_balance + interval;
  3378. update_next_balance = 1;
  3379. }
  3380. /*
  3381. * Stop the load balance at this level. There is another
  3382. * CPU in our sched group which is doing load balancing more
  3383. * actively.
  3384. */
  3385. if (!balance)
  3386. break;
  3387. }
  3388. /*
  3389. * next_balance will be updated only when there is a need.
  3390. * When the cpu is attached to null domain for ex, it will not be
  3391. * updated.
  3392. */
  3393. if (likely(update_next_balance))
  3394. rq->next_balance = next_balance;
  3395. free_cpumask_var(tmp);
  3396. }
  3397. /*
  3398. * run_rebalance_domains is triggered when needed from the scheduler tick.
  3399. * In CONFIG_NO_HZ case, the idle load balance owner will do the
  3400. * rebalancing for all the cpus for whom scheduler ticks are stopped.
  3401. */
  3402. static void run_rebalance_domains(struct softirq_action *h)
  3403. {
  3404. int this_cpu = smp_processor_id();
  3405. struct rq *this_rq = cpu_rq(this_cpu);
  3406. enum cpu_idle_type idle = this_rq->idle_at_tick ?
  3407. CPU_IDLE : CPU_NOT_IDLE;
  3408. rebalance_domains(this_cpu, idle);
  3409. #ifdef CONFIG_NO_HZ
  3410. /*
  3411. * If this cpu is the owner for idle load balancing, then do the
  3412. * balancing on behalf of the other idle cpus whose ticks are
  3413. * stopped.
  3414. */
  3415. if (this_rq->idle_at_tick &&
  3416. atomic_read(&nohz.load_balancer) == this_cpu) {
  3417. struct rq *rq;
  3418. int balance_cpu;
  3419. for_each_cpu(balance_cpu, nohz.cpu_mask) {
  3420. if (balance_cpu == this_cpu)
  3421. continue;
  3422. /*
  3423. * If this cpu gets work to do, stop the load balancing
  3424. * work being done for other cpus. Next load
  3425. * balancing owner will pick it up.
  3426. */
  3427. if (need_resched())
  3428. break;
  3429. rebalance_domains(balance_cpu, CPU_IDLE);
  3430. rq = cpu_rq(balance_cpu);
  3431. if (time_after(this_rq->next_balance, rq->next_balance))
  3432. this_rq->next_balance = rq->next_balance;
  3433. }
  3434. }
  3435. #endif
  3436. }
  3437. /*
  3438. * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
  3439. *
  3440. * In case of CONFIG_NO_HZ, this is the place where we nominate a new
  3441. * idle load balancing owner or decide to stop the periodic load balancing,
  3442. * if the whole system is idle.
  3443. */
  3444. static inline void trigger_load_balance(struct rq *rq, int cpu)
  3445. {
  3446. #ifdef CONFIG_NO_HZ
  3447. /*
  3448. * If we were in the nohz mode recently and busy at the current
  3449. * scheduler tick, then check if we need to nominate new idle
  3450. * load balancer.
  3451. */
  3452. if (rq->in_nohz_recently && !rq->idle_at_tick) {
  3453. rq->in_nohz_recently = 0;
  3454. if (atomic_read(&nohz.load_balancer) == cpu) {
  3455. cpumask_clear_cpu(cpu, nohz.cpu_mask);
  3456. atomic_set(&nohz.load_balancer, -1);
  3457. }
  3458. if (atomic_read(&nohz.load_balancer) == -1) {
  3459. /*
  3460. * simple selection for now: Nominate the
  3461. * first cpu in the nohz list to be the next
  3462. * ilb owner.
  3463. *
  3464. * TBD: Traverse the sched domains and nominate
  3465. * the nearest cpu in the nohz.cpu_mask.
  3466. */
  3467. int ilb = cpumask_first(nohz.cpu_mask);
  3468. if (ilb < nr_cpu_ids)
  3469. resched_cpu(ilb);
  3470. }
  3471. }
  3472. /*
  3473. * If this cpu is idle and doing idle load balancing for all the
  3474. * cpus with ticks stopped, is it time for that to stop?
  3475. */
  3476. if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
  3477. cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
  3478. resched_cpu(cpu);
  3479. return;
  3480. }
  3481. /*
  3482. * If this cpu is idle and the idle load balancing is done by
  3483. * someone else, then no need raise the SCHED_SOFTIRQ
  3484. */
  3485. if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
  3486. cpumask_test_cpu(cpu, nohz.cpu_mask))
  3487. return;
  3488. #endif
  3489. if (time_after_eq(jiffies, rq->next_balance))
  3490. raise_softirq(SCHED_SOFTIRQ);
  3491. }
  3492. #else /* CONFIG_SMP */
  3493. /*
  3494. * on UP we do not need to balance between CPUs:
  3495. */
  3496. static inline void idle_balance(int cpu, struct rq *rq)
  3497. {
  3498. }
  3499. #endif
  3500. DEFINE_PER_CPU(struct kernel_stat, kstat);
  3501. EXPORT_PER_CPU_SYMBOL(kstat);
  3502. /*
  3503. * Return any ns on the sched_clock that have not yet been banked in
  3504. * @p in case that task is currently running.
  3505. */
  3506. unsigned long long task_delta_exec(struct task_struct *p)
  3507. {
  3508. unsigned long flags;
  3509. struct rq *rq;
  3510. u64 ns = 0;
  3511. rq = task_rq_lock(p, &flags);
  3512. if (task_current(rq, p)) {
  3513. u64 delta_exec;
  3514. update_rq_clock(rq);
  3515. delta_exec = rq->clock - p->se.exec_start;
  3516. if ((s64)delta_exec > 0)
  3517. ns = delta_exec;
  3518. }
  3519. task_rq_unlock(rq, &flags);
  3520. return ns;
  3521. }
  3522. /*
  3523. * Account user cpu time to a process.
  3524. * @p: the process that the cpu time gets accounted to
  3525. * @cputime: the cpu time spent in user space since the last update
  3526. * @cputime_scaled: cputime scaled by cpu frequency
  3527. */
  3528. void account_user_time(struct task_struct *p, cputime_t cputime,
  3529. cputime_t cputime_scaled)
  3530. {
  3531. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3532. cputime64_t tmp;
  3533. /* Add user time to process. */
  3534. p->utime = cputime_add(p->utime, cputime);
  3535. p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
  3536. account_group_user_time(p, cputime);
  3537. /* Add user time to cpustat. */
  3538. tmp = cputime_to_cputime64(cputime);
  3539. if (TASK_NICE(p) > 0)
  3540. cpustat->nice = cputime64_add(cpustat->nice, tmp);
  3541. else
  3542. cpustat->user = cputime64_add(cpustat->user, tmp);
  3543. /* Account for user time used */
  3544. acct_update_integrals(p);
  3545. }
  3546. /*
  3547. * Account guest cpu time to a process.
  3548. * @p: the process that the cpu time gets accounted to
  3549. * @cputime: the cpu time spent in virtual machine since the last update
  3550. * @cputime_scaled: cputime scaled by cpu frequency
  3551. */
  3552. static void account_guest_time(struct task_struct *p, cputime_t cputime,
  3553. cputime_t cputime_scaled)
  3554. {
  3555. cputime64_t tmp;
  3556. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3557. tmp = cputime_to_cputime64(cputime);
  3558. /* Add guest time to process. */
  3559. p->utime = cputime_add(p->utime, cputime);
  3560. p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
  3561. account_group_user_time(p, cputime);
  3562. p->gtime = cputime_add(p->gtime, cputime);
  3563. /* Add guest time to cpustat. */
  3564. cpustat->user = cputime64_add(cpustat->user, tmp);
  3565. cpustat->guest = cputime64_add(cpustat->guest, tmp);
  3566. }
  3567. /*
  3568. * Account system cpu time to a process.
  3569. * @p: the process that the cpu time gets accounted to
  3570. * @hardirq_offset: the offset to subtract from hardirq_count()
  3571. * @cputime: the cpu time spent in kernel space since the last update
  3572. * @cputime_scaled: cputime scaled by cpu frequency
  3573. */
  3574. void account_system_time(struct task_struct *p, int hardirq_offset,
  3575. cputime_t cputime, cputime_t cputime_scaled)
  3576. {
  3577. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3578. cputime64_t tmp;
  3579. if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
  3580. account_guest_time(p, cputime, cputime_scaled);
  3581. return;
  3582. }
  3583. /* Add system time to process. */
  3584. p->stime = cputime_add(p->stime, cputime);
  3585. p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
  3586. account_group_system_time(p, cputime);
  3587. /* Add system time to cpustat. */
  3588. tmp = cputime_to_cputime64(cputime);
  3589. if (hardirq_count() - hardirq_offset)
  3590. cpustat->irq = cputime64_add(cpustat->irq, tmp);
  3591. else if (softirq_count())
  3592. cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
  3593. else
  3594. cpustat->system = cputime64_add(cpustat->system, tmp);
  3595. /* Account for system time used */
  3596. acct_update_integrals(p);
  3597. }
  3598. /*
  3599. * Account for involuntary wait time.
  3600. * @steal: the cpu time spent in involuntary wait
  3601. */
  3602. void account_steal_time(cputime_t cputime)
  3603. {
  3604. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3605. cputime64_t cputime64 = cputime_to_cputime64(cputime);
  3606. cpustat->steal = cputime64_add(cpustat->steal, cputime64);
  3607. }
  3608. /*
  3609. * Account for idle time.
  3610. * @cputime: the cpu time spent in idle wait
  3611. */
  3612. void account_idle_time(cputime_t cputime)
  3613. {
  3614. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3615. cputime64_t cputime64 = cputime_to_cputime64(cputime);
  3616. struct rq *rq = this_rq();
  3617. if (atomic_read(&rq->nr_iowait) > 0)
  3618. cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
  3619. else
  3620. cpustat->idle = cputime64_add(cpustat->idle, cputime64);
  3621. }
  3622. #ifndef CONFIG_VIRT_CPU_ACCOUNTING
  3623. /*
  3624. * Account a single tick of cpu time.
  3625. * @p: the process that the cpu time gets accounted to
  3626. * @user_tick: indicates if the tick is a user or a system tick
  3627. */
  3628. void account_process_tick(struct task_struct *p, int user_tick)
  3629. {
  3630. cputime_t one_jiffy = jiffies_to_cputime(1);
  3631. cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy);
  3632. struct rq *rq = this_rq();
  3633. if (user_tick)
  3634. account_user_time(p, one_jiffy, one_jiffy_scaled);
  3635. else if (p != rq->idle)
  3636. account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
  3637. one_jiffy_scaled);
  3638. else
  3639. account_idle_time(one_jiffy);
  3640. }
  3641. /*
  3642. * Account multiple ticks of steal time.
  3643. * @p: the process from which the cpu time has been stolen
  3644. * @ticks: number of stolen ticks
  3645. */
  3646. void account_steal_ticks(unsigned long ticks)
  3647. {
  3648. account_steal_time(jiffies_to_cputime(ticks));
  3649. }
  3650. /*
  3651. * Account multiple ticks of idle time.
  3652. * @ticks: number of stolen ticks
  3653. */
  3654. void account_idle_ticks(unsigned long ticks)
  3655. {
  3656. account_idle_time(jiffies_to_cputime(ticks));
  3657. }
  3658. #endif
  3659. /*
  3660. * Use precise platform statistics if available:
  3661. */
  3662. #ifdef CONFIG_VIRT_CPU_ACCOUNTING
  3663. cputime_t task_utime(struct task_struct *p)
  3664. {
  3665. return p->utime;
  3666. }
  3667. cputime_t task_stime(struct task_struct *p)
  3668. {
  3669. return p->stime;
  3670. }
  3671. #else
  3672. cputime_t task_utime(struct task_struct *p)
  3673. {
  3674. clock_t utime = cputime_to_clock_t(p->utime),
  3675. total = utime + cputime_to_clock_t(p->stime);
  3676. u64 temp;
  3677. /*
  3678. * Use CFS's precise accounting:
  3679. */
  3680. temp = (u64)nsec_to_clock_t(p->se.sum_exec_runtime);
  3681. if (total) {
  3682. temp *= utime;
  3683. do_div(temp, total);
  3684. }
  3685. utime = (clock_t)temp;
  3686. p->prev_utime = max(p->prev_utime, clock_t_to_cputime(utime));
  3687. return p->prev_utime;
  3688. }
  3689. cputime_t task_stime(struct task_struct *p)
  3690. {
  3691. clock_t stime;
  3692. /*
  3693. * Use CFS's precise accounting. (we subtract utime from
  3694. * the total, to make sure the total observed by userspace
  3695. * grows monotonically - apps rely on that):
  3696. */
  3697. stime = nsec_to_clock_t(p->se.sum_exec_runtime) -
  3698. cputime_to_clock_t(task_utime(p));
  3699. if (stime >= 0)
  3700. p->prev_stime = max(p->prev_stime, clock_t_to_cputime(stime));
  3701. return p->prev_stime;
  3702. }
  3703. #endif
  3704. inline cputime_t task_gtime(struct task_struct *p)
  3705. {
  3706. return p->gtime;
  3707. }
  3708. /*
  3709. * This function gets called by the timer code, with HZ frequency.
  3710. * We call it with interrupts disabled.
  3711. *
  3712. * It also gets called by the fork code, when changing the parent's
  3713. * timeslices.
  3714. */
  3715. void scheduler_tick(void)
  3716. {
  3717. int cpu = smp_processor_id();
  3718. struct rq *rq = cpu_rq(cpu);
  3719. struct task_struct *curr = rq->curr;
  3720. sched_clock_tick();
  3721. spin_lock(&rq->lock);
  3722. update_rq_clock(rq);
  3723. update_cpu_load(rq);
  3724. curr->sched_class->task_tick(rq, curr, 0);
  3725. spin_unlock(&rq->lock);
  3726. #ifdef CONFIG_SMP
  3727. rq->idle_at_tick = idle_cpu(cpu);
  3728. trigger_load_balance(rq, cpu);
  3729. #endif
  3730. }
  3731. #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
  3732. defined(CONFIG_PREEMPT_TRACER))
  3733. static inline unsigned long get_parent_ip(unsigned long addr)
  3734. {
  3735. if (in_lock_functions(addr)) {
  3736. addr = CALLER_ADDR2;
  3737. if (in_lock_functions(addr))
  3738. addr = CALLER_ADDR3;
  3739. }
  3740. return addr;
  3741. }
  3742. void __kprobes add_preempt_count(int val)
  3743. {
  3744. #ifdef CONFIG_DEBUG_PREEMPT
  3745. /*
  3746. * Underflow?
  3747. */
  3748. if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
  3749. return;
  3750. #endif
  3751. preempt_count() += val;
  3752. #ifdef CONFIG_DEBUG_PREEMPT
  3753. /*
  3754. * Spinlock count overflowing soon?
  3755. */
  3756. DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
  3757. PREEMPT_MASK - 10);
  3758. #endif
  3759. if (preempt_count() == val)
  3760. trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
  3761. }
  3762. EXPORT_SYMBOL(add_preempt_count);
  3763. void __kprobes sub_preempt_count(int val)
  3764. {
  3765. #ifdef CONFIG_DEBUG_PREEMPT
  3766. /*
  3767. * Underflow?
  3768. */
  3769. if (DEBUG_LOCKS_WARN_ON(val > preempt_count() - (!!kernel_locked())))
  3770. return;
  3771. /*
  3772. * Is the spinlock portion underflowing?
  3773. */
  3774. if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
  3775. !(preempt_count() & PREEMPT_MASK)))
  3776. return;
  3777. #endif
  3778. if (preempt_count() == val)
  3779. trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
  3780. preempt_count() -= val;
  3781. }
  3782. EXPORT_SYMBOL(sub_preempt_count);
  3783. #endif
  3784. /*
  3785. * Print scheduling while atomic bug:
  3786. */
  3787. static noinline void __schedule_bug(struct task_struct *prev)
  3788. {
  3789. struct pt_regs *regs = get_irq_regs();
  3790. printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
  3791. prev->comm, prev->pid, preempt_count());
  3792. debug_show_held_locks(prev);
  3793. print_modules();
  3794. if (irqs_disabled())
  3795. print_irqtrace_events(prev);
  3796. if (regs)
  3797. show_regs(regs);
  3798. else
  3799. dump_stack();
  3800. }
  3801. /*
  3802. * Various schedule()-time debugging checks and statistics:
  3803. */
  3804. static inline void schedule_debug(struct task_struct *prev)
  3805. {
  3806. /*
  3807. * Test if we are atomic. Since do_exit() needs to call into
  3808. * schedule() atomically, we ignore that path for now.
  3809. * Otherwise, whine if we are scheduling when we should not be.
  3810. */
  3811. if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
  3812. __schedule_bug(prev);
  3813. profile_hit(SCHED_PROFILING, __builtin_return_address(0));
  3814. schedstat_inc(this_rq(), sched_count);
  3815. #ifdef CONFIG_SCHEDSTATS
  3816. if (unlikely(prev->lock_depth >= 0)) {
  3817. schedstat_inc(this_rq(), bkl_count);
  3818. schedstat_inc(prev, sched_info.bkl_count);
  3819. }
  3820. #endif
  3821. }
  3822. /*
  3823. * Pick up the highest-prio task:
  3824. */
  3825. static inline struct task_struct *
  3826. pick_next_task(struct rq *rq, struct task_struct *prev)
  3827. {
  3828. const struct sched_class *class;
  3829. struct task_struct *p;
  3830. /*
  3831. * Optimization: we know that if all tasks are in
  3832. * the fair class we can call that function directly:
  3833. */
  3834. if (likely(rq->nr_running == rq->cfs.nr_running)) {
  3835. p = fair_sched_class.pick_next_task(rq);
  3836. if (likely(p))
  3837. return p;
  3838. }
  3839. class = sched_class_highest;
  3840. for ( ; ; ) {
  3841. p = class->pick_next_task(rq);
  3842. if (p)
  3843. return p;
  3844. /*
  3845. * Will never be NULL as the idle class always
  3846. * returns a non-NULL p:
  3847. */
  3848. class = class->next;
  3849. }
  3850. }
  3851. /*
  3852. * schedule() is the main scheduler function.
  3853. */
  3854. asmlinkage void __sched schedule(void)
  3855. {
  3856. struct task_struct *prev, *next;
  3857. unsigned long *switch_count;
  3858. struct rq *rq;
  3859. int cpu;
  3860. need_resched:
  3861. preempt_disable();
  3862. cpu = smp_processor_id();
  3863. rq = cpu_rq(cpu);
  3864. rcu_qsctr_inc(cpu);
  3865. prev = rq->curr;
  3866. switch_count = &prev->nivcsw;
  3867. release_kernel_lock(prev);
  3868. need_resched_nonpreemptible:
  3869. schedule_debug(prev);
  3870. if (sched_feat(HRTICK))
  3871. hrtick_clear(rq);
  3872. spin_lock_irq(&rq->lock);
  3873. update_rq_clock(rq);
  3874. clear_tsk_need_resched(prev);
  3875. if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
  3876. if (unlikely(signal_pending_state(prev->state, prev)))
  3877. prev->state = TASK_RUNNING;
  3878. else
  3879. deactivate_task(rq, prev, 1);
  3880. switch_count = &prev->nvcsw;
  3881. }
  3882. #ifdef CONFIG_SMP
  3883. if (prev->sched_class->pre_schedule)
  3884. prev->sched_class->pre_schedule(rq, prev);
  3885. #endif
  3886. if (unlikely(!rq->nr_running))
  3887. idle_balance(cpu, rq);
  3888. prev->sched_class->put_prev_task(rq, prev);
  3889. next = pick_next_task(rq, prev);
  3890. if (likely(prev != next)) {
  3891. sched_info_switch(prev, next);
  3892. rq->nr_switches++;
  3893. rq->curr = next;
  3894. ++*switch_count;
  3895. context_switch(rq, prev, next); /* unlocks the rq */
  3896. /*
  3897. * the context switch might have flipped the stack from under
  3898. * us, hence refresh the local variables.
  3899. */
  3900. cpu = smp_processor_id();
  3901. rq = cpu_rq(cpu);
  3902. } else
  3903. spin_unlock_irq(&rq->lock);
  3904. if (unlikely(reacquire_kernel_lock(current) < 0))
  3905. goto need_resched_nonpreemptible;
  3906. preempt_enable_no_resched();
  3907. if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
  3908. goto need_resched;
  3909. }
  3910. EXPORT_SYMBOL(schedule);
  3911. #ifdef CONFIG_PREEMPT
  3912. /*
  3913. * this is the entry point to schedule() from in-kernel preemption
  3914. * off of preempt_enable. Kernel preemptions off return from interrupt
  3915. * occur there and call schedule directly.
  3916. */
  3917. asmlinkage void __sched preempt_schedule(void)
  3918. {
  3919. struct thread_info *ti = current_thread_info();
  3920. /*
  3921. * If there is a non-zero preempt_count or interrupts are disabled,
  3922. * we do not want to preempt the current task. Just return..
  3923. */
  3924. if (likely(ti->preempt_count || irqs_disabled()))
  3925. return;
  3926. do {
  3927. add_preempt_count(PREEMPT_ACTIVE);
  3928. schedule();
  3929. sub_preempt_count(PREEMPT_ACTIVE);
  3930. /*
  3931. * Check again in case we missed a preemption opportunity
  3932. * between schedule and now.
  3933. */
  3934. barrier();
  3935. } while (unlikely(test_thread_flag(TIF_NEED_RESCHED)));
  3936. }
  3937. EXPORT_SYMBOL(preempt_schedule);
  3938. /*
  3939. * this is the entry point to schedule() from kernel preemption
  3940. * off of irq context.
  3941. * Note, that this is called and return with irqs disabled. This will
  3942. * protect us against recursive calling from irq.
  3943. */
  3944. asmlinkage void __sched preempt_schedule_irq(void)
  3945. {
  3946. struct thread_info *ti = current_thread_info();
  3947. /* Catch callers which need to be fixed */
  3948. BUG_ON(ti->preempt_count || !irqs_disabled());
  3949. do {
  3950. add_preempt_count(PREEMPT_ACTIVE);
  3951. local_irq_enable();
  3952. schedule();
  3953. local_irq_disable();
  3954. sub_preempt_count(PREEMPT_ACTIVE);
  3955. /*
  3956. * Check again in case we missed a preemption opportunity
  3957. * between schedule and now.
  3958. */
  3959. barrier();
  3960. } while (unlikely(test_thread_flag(TIF_NEED_RESCHED)));
  3961. }
  3962. #endif /* CONFIG_PREEMPT */
  3963. int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
  3964. void *key)
  3965. {
  3966. return try_to_wake_up(curr->private, mode, sync);
  3967. }
  3968. EXPORT_SYMBOL(default_wake_function);
  3969. /*
  3970. * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
  3971. * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
  3972. * number) then we wake all the non-exclusive tasks and one exclusive task.
  3973. *
  3974. * There are circumstances in which we can try to wake a task which has already
  3975. * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
  3976. * zero in this (rare) case, and we handle it by continuing to scan the queue.
  3977. */
  3978. static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
  3979. int nr_exclusive, int sync, void *key)
  3980. {
  3981. wait_queue_t *curr, *next;
  3982. list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
  3983. unsigned flags = curr->flags;
  3984. if (curr->func(curr, mode, sync, key) &&
  3985. (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
  3986. break;
  3987. }
  3988. }
  3989. /**
  3990. * __wake_up - wake up threads blocked on a waitqueue.
  3991. * @q: the waitqueue
  3992. * @mode: which threads
  3993. * @nr_exclusive: how many wake-one or wake-many threads to wake up
  3994. * @key: is directly passed to the wakeup function
  3995. */
  3996. void __wake_up(wait_queue_head_t *q, unsigned int mode,
  3997. int nr_exclusive, void *key)
  3998. {
  3999. unsigned long flags;
  4000. spin_lock_irqsave(&q->lock, flags);
  4001. __wake_up_common(q, mode, nr_exclusive, 0, key);
  4002. spin_unlock_irqrestore(&q->lock, flags);
  4003. }
  4004. EXPORT_SYMBOL(__wake_up);
  4005. /*
  4006. * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
  4007. */
  4008. void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
  4009. {
  4010. __wake_up_common(q, mode, 1, 0, NULL);
  4011. }
  4012. /**
  4013. * __wake_up_sync - wake up threads blocked on a waitqueue.
  4014. * @q: the waitqueue
  4015. * @mode: which threads
  4016. * @nr_exclusive: how many wake-one or wake-many threads to wake up
  4017. *
  4018. * The sync wakeup differs that the waker knows that it will schedule
  4019. * away soon, so while the target thread will be woken up, it will not
  4020. * be migrated to another CPU - ie. the two threads are 'synchronized'
  4021. * with each other. This can prevent needless bouncing between CPUs.
  4022. *
  4023. * On UP it can prevent extra preemption.
  4024. */
  4025. void
  4026. __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
  4027. {
  4028. unsigned long flags;
  4029. int sync = 1;
  4030. if (unlikely(!q))
  4031. return;
  4032. if (unlikely(!nr_exclusive))
  4033. sync = 0;
  4034. spin_lock_irqsave(&q->lock, flags);
  4035. __wake_up_common(q, mode, nr_exclusive, sync, NULL);
  4036. spin_unlock_irqrestore(&q->lock, flags);
  4037. }
  4038. EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */
  4039. /**
  4040. * complete: - signals a single thread waiting on this completion
  4041. * @x: holds the state of this particular completion
  4042. *
  4043. * This will wake up a single thread waiting on this completion. Threads will be
  4044. * awakened in the same order in which they were queued.
  4045. *
  4046. * See also complete_all(), wait_for_completion() and related routines.
  4047. */
  4048. void complete(struct completion *x)
  4049. {
  4050. unsigned long flags;
  4051. spin_lock_irqsave(&x->wait.lock, flags);
  4052. x->done++;
  4053. __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
  4054. spin_unlock_irqrestore(&x->wait.lock, flags);
  4055. }
  4056. EXPORT_SYMBOL(complete);
  4057. /**
  4058. * complete_all: - signals all threads waiting on this completion
  4059. * @x: holds the state of this particular completion
  4060. *
  4061. * This will wake up all threads waiting on this particular completion event.
  4062. */
  4063. void complete_all(struct completion *x)
  4064. {
  4065. unsigned long flags;
  4066. spin_lock_irqsave(&x->wait.lock, flags);
  4067. x->done += UINT_MAX/2;
  4068. __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
  4069. spin_unlock_irqrestore(&x->wait.lock, flags);
  4070. }
  4071. EXPORT_SYMBOL(complete_all);
  4072. static inline long __sched
  4073. do_wait_for_common(struct completion *x, long timeout, int state)
  4074. {
  4075. if (!x->done) {
  4076. DECLARE_WAITQUEUE(wait, current);
  4077. wait.flags |= WQ_FLAG_EXCLUSIVE;
  4078. __add_wait_queue_tail(&x->wait, &wait);
  4079. do {
  4080. if (signal_pending_state(state, current)) {
  4081. timeout = -ERESTARTSYS;
  4082. break;
  4083. }
  4084. __set_current_state(state);
  4085. spin_unlock_irq(&x->wait.lock);
  4086. timeout = schedule_timeout(timeout);
  4087. spin_lock_irq(&x->wait.lock);
  4088. } while (!x->done && timeout);
  4089. __remove_wait_queue(&x->wait, &wait);
  4090. if (!x->done)
  4091. return timeout;
  4092. }
  4093. x->done--;
  4094. return timeout ?: 1;
  4095. }
  4096. static long __sched
  4097. wait_for_common(struct completion *x, long timeout, int state)
  4098. {
  4099. might_sleep();
  4100. spin_lock_irq(&x->wait.lock);
  4101. timeout = do_wait_for_common(x, timeout, state);
  4102. spin_unlock_irq(&x->wait.lock);
  4103. return timeout;
  4104. }
  4105. /**
  4106. * wait_for_completion: - waits for completion of a task
  4107. * @x: holds the state of this particular completion
  4108. *
  4109. * This waits to be signaled for completion of a specific task. It is NOT
  4110. * interruptible and there is no timeout.
  4111. *
  4112. * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
  4113. * and interrupt capability. Also see complete().
  4114. */
  4115. void __sched wait_for_completion(struct completion *x)
  4116. {
  4117. wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
  4118. }
  4119. EXPORT_SYMBOL(wait_for_completion);
  4120. /**
  4121. * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
  4122. * @x: holds the state of this particular completion
  4123. * @timeout: timeout value in jiffies
  4124. *
  4125. * This waits for either a completion of a specific task to be signaled or for a
  4126. * specified timeout to expire. The timeout is in jiffies. It is not
  4127. * interruptible.
  4128. */
  4129. unsigned long __sched
  4130. wait_for_completion_timeout(struct completion *x, unsigned long timeout)
  4131. {
  4132. return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
  4133. }
  4134. EXPORT_SYMBOL(wait_for_completion_timeout);
  4135. /**
  4136. * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
  4137. * @x: holds the state of this particular completion
  4138. *
  4139. * This waits for completion of a specific task to be signaled. It is
  4140. * interruptible.
  4141. */
  4142. int __sched wait_for_completion_interruptible(struct completion *x)
  4143. {
  4144. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
  4145. if (t == -ERESTARTSYS)
  4146. return t;
  4147. return 0;
  4148. }
  4149. EXPORT_SYMBOL(wait_for_completion_interruptible);
  4150. /**
  4151. * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
  4152. * @x: holds the state of this particular completion
  4153. * @timeout: timeout value in jiffies
  4154. *
  4155. * This waits for either a completion of a specific task to be signaled or for a
  4156. * specified timeout to expire. It is interruptible. The timeout is in jiffies.
  4157. */
  4158. unsigned long __sched
  4159. wait_for_completion_interruptible_timeout(struct completion *x,
  4160. unsigned long timeout)
  4161. {
  4162. return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
  4163. }
  4164. EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
  4165. /**
  4166. * wait_for_completion_killable: - waits for completion of a task (killable)
  4167. * @x: holds the state of this particular completion
  4168. *
  4169. * This waits to be signaled for completion of a specific task. It can be
  4170. * interrupted by a kill signal.
  4171. */
  4172. int __sched wait_for_completion_killable(struct completion *x)
  4173. {
  4174. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
  4175. if (t == -ERESTARTSYS)
  4176. return t;
  4177. return 0;
  4178. }
  4179. EXPORT_SYMBOL(wait_for_completion_killable);
  4180. /**
  4181. * try_wait_for_completion - try to decrement a completion without blocking
  4182. * @x: completion structure
  4183. *
  4184. * Returns: 0 if a decrement cannot be done without blocking
  4185. * 1 if a decrement succeeded.
  4186. *
  4187. * If a completion is being used as a counting completion,
  4188. * attempt to decrement the counter without blocking. This
  4189. * enables us to avoid waiting if the resource the completion
  4190. * is protecting is not available.
  4191. */
  4192. bool try_wait_for_completion(struct completion *x)
  4193. {
  4194. int ret = 1;
  4195. spin_lock_irq(&x->wait.lock);
  4196. if (!x->done)
  4197. ret = 0;
  4198. else
  4199. x->done--;
  4200. spin_unlock_irq(&x->wait.lock);
  4201. return ret;
  4202. }
  4203. EXPORT_SYMBOL(try_wait_for_completion);
  4204. /**
  4205. * completion_done - Test to see if a completion has any waiters
  4206. * @x: completion structure
  4207. *
  4208. * Returns: 0 if there are waiters (wait_for_completion() in progress)
  4209. * 1 if there are no waiters.
  4210. *
  4211. */
  4212. bool completion_done(struct completion *x)
  4213. {
  4214. int ret = 1;
  4215. spin_lock_irq(&x->wait.lock);
  4216. if (!x->done)
  4217. ret = 0;
  4218. spin_unlock_irq(&x->wait.lock);
  4219. return ret;
  4220. }
  4221. EXPORT_SYMBOL(completion_done);
  4222. static long __sched
  4223. sleep_on_common(wait_queue_head_t *q, int state, long timeout)
  4224. {
  4225. unsigned long flags;
  4226. wait_queue_t wait;
  4227. init_waitqueue_entry(&wait, current);
  4228. __set_current_state(state);
  4229. spin_lock_irqsave(&q->lock, flags);
  4230. __add_wait_queue(q, &wait);
  4231. spin_unlock(&q->lock);
  4232. timeout = schedule_timeout(timeout);
  4233. spin_lock_irq(&q->lock);
  4234. __remove_wait_queue(q, &wait);
  4235. spin_unlock_irqrestore(&q->lock, flags);
  4236. return timeout;
  4237. }
  4238. void __sched interruptible_sleep_on(wait_queue_head_t *q)
  4239. {
  4240. sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
  4241. }
  4242. EXPORT_SYMBOL(interruptible_sleep_on);
  4243. long __sched
  4244. interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
  4245. {
  4246. return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
  4247. }
  4248. EXPORT_SYMBOL(interruptible_sleep_on_timeout);
  4249. void __sched sleep_on(wait_queue_head_t *q)
  4250. {
  4251. sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
  4252. }
  4253. EXPORT_SYMBOL(sleep_on);
  4254. long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
  4255. {
  4256. return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
  4257. }
  4258. EXPORT_SYMBOL(sleep_on_timeout);
  4259. #ifdef CONFIG_RT_MUTEXES
  4260. /*
  4261. * rt_mutex_setprio - set the current priority of a task
  4262. * @p: task
  4263. * @prio: prio value (kernel-internal form)
  4264. *
  4265. * This function changes the 'effective' priority of a task. It does
  4266. * not touch ->normal_prio like __setscheduler().
  4267. *
  4268. * Used by the rt_mutex code to implement priority inheritance logic.
  4269. */
  4270. void rt_mutex_setprio(struct task_struct *p, int prio)
  4271. {
  4272. unsigned long flags;
  4273. int oldprio, on_rq, running;
  4274. struct rq *rq;
  4275. const struct sched_class *prev_class = p->sched_class;
  4276. BUG_ON(prio < 0 || prio > MAX_PRIO);
  4277. rq = task_rq_lock(p, &flags);
  4278. update_rq_clock(rq);
  4279. oldprio = p->prio;
  4280. on_rq = p->se.on_rq;
  4281. running = task_current(rq, p);
  4282. if (on_rq)
  4283. dequeue_task(rq, p, 0);
  4284. if (running)
  4285. p->sched_class->put_prev_task(rq, p);
  4286. if (rt_prio(prio))
  4287. p->sched_class = &rt_sched_class;
  4288. else
  4289. p->sched_class = &fair_sched_class;
  4290. p->prio = prio;
  4291. if (running)
  4292. p->sched_class->set_curr_task(rq);
  4293. if (on_rq) {
  4294. enqueue_task(rq, p, 0);
  4295. check_class_changed(rq, p, prev_class, oldprio, running);
  4296. }
  4297. task_rq_unlock(rq, &flags);
  4298. }
  4299. #endif
  4300. void set_user_nice(struct task_struct *p, long nice)
  4301. {
  4302. int old_prio, delta, on_rq;
  4303. unsigned long flags;
  4304. struct rq *rq;
  4305. if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
  4306. return;
  4307. /*
  4308. * We have to be careful, if called from sys_setpriority(),
  4309. * the task might be in the middle of scheduling on another CPU.
  4310. */
  4311. rq = task_rq_lock(p, &flags);
  4312. update_rq_clock(rq);
  4313. /*
  4314. * The RT priorities are set via sched_setscheduler(), but we still
  4315. * allow the 'normal' nice value to be set - but as expected
  4316. * it wont have any effect on scheduling until the task is
  4317. * SCHED_FIFO/SCHED_RR:
  4318. */
  4319. if (task_has_rt_policy(p)) {
  4320. p->static_prio = NICE_TO_PRIO(nice);
  4321. goto out_unlock;
  4322. }
  4323. on_rq = p->se.on_rq;
  4324. if (on_rq)
  4325. dequeue_task(rq, p, 0);
  4326. p->static_prio = NICE_TO_PRIO(nice);
  4327. set_load_weight(p);
  4328. old_prio = p->prio;
  4329. p->prio = effective_prio(p);
  4330. delta = p->prio - old_prio;
  4331. if (on_rq) {
  4332. enqueue_task(rq, p, 0);
  4333. /*
  4334. * If the task increased its priority or is running and
  4335. * lowered its priority, then reschedule its CPU:
  4336. */
  4337. if (delta < 0 || (delta > 0 && task_running(rq, p)))
  4338. resched_task(rq->curr);
  4339. }
  4340. out_unlock:
  4341. task_rq_unlock(rq, &flags);
  4342. }
  4343. EXPORT_SYMBOL(set_user_nice);
  4344. /*
  4345. * can_nice - check if a task can reduce its nice value
  4346. * @p: task
  4347. * @nice: nice value
  4348. */
  4349. int can_nice(const struct task_struct *p, const int nice)
  4350. {
  4351. /* convert nice value [19,-20] to rlimit style value [1,40] */
  4352. int nice_rlim = 20 - nice;
  4353. return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur ||
  4354. capable(CAP_SYS_NICE));
  4355. }
  4356. #ifdef __ARCH_WANT_SYS_NICE
  4357. /*
  4358. * sys_nice - change the priority of the current process.
  4359. * @increment: priority increment
  4360. *
  4361. * sys_setpriority is a more generic, but much slower function that
  4362. * does similar things.
  4363. */
  4364. asmlinkage long sys_nice(int increment)
  4365. {
  4366. long nice, retval;
  4367. /*
  4368. * Setpriority might change our priority at the same moment.
  4369. * We don't have to worry. Conceptually one call occurs first
  4370. * and we have a single winner.
  4371. */
  4372. if (increment < -40)
  4373. increment = -40;
  4374. if (increment > 40)
  4375. increment = 40;
  4376. nice = PRIO_TO_NICE(current->static_prio) + increment;
  4377. if (nice < -20)
  4378. nice = -20;
  4379. if (nice > 19)
  4380. nice = 19;
  4381. if (increment < 0 && !can_nice(current, nice))
  4382. return -EPERM;
  4383. retval = security_task_setnice(current, nice);
  4384. if (retval)
  4385. return retval;
  4386. set_user_nice(current, nice);
  4387. return 0;
  4388. }
  4389. #endif
  4390. /**
  4391. * task_prio - return the priority value of a given task.
  4392. * @p: the task in question.
  4393. *
  4394. * This is the priority value as seen by users in /proc.
  4395. * RT tasks are offset by -200. Normal tasks are centered
  4396. * around 0, value goes from -16 to +15.
  4397. */
  4398. int task_prio(const struct task_struct *p)
  4399. {
  4400. return p->prio - MAX_RT_PRIO;
  4401. }
  4402. /**
  4403. * task_nice - return the nice value of a given task.
  4404. * @p: the task in question.
  4405. */
  4406. int task_nice(const struct task_struct *p)
  4407. {
  4408. return TASK_NICE(p);
  4409. }
  4410. EXPORT_SYMBOL(task_nice);
  4411. /**
  4412. * idle_cpu - is a given cpu idle currently?
  4413. * @cpu: the processor in question.
  4414. */
  4415. int idle_cpu(int cpu)
  4416. {
  4417. return cpu_curr(cpu) == cpu_rq(cpu)->idle;
  4418. }
  4419. /**
  4420. * idle_task - return the idle task for a given cpu.
  4421. * @cpu: the processor in question.
  4422. */
  4423. struct task_struct *idle_task(int cpu)
  4424. {
  4425. return cpu_rq(cpu)->idle;
  4426. }
  4427. /**
  4428. * find_process_by_pid - find a process with a matching PID value.
  4429. * @pid: the pid in question.
  4430. */
  4431. static struct task_struct *find_process_by_pid(pid_t pid)
  4432. {
  4433. return pid ? find_task_by_vpid(pid) : current;
  4434. }
  4435. /* Actually do priority change: must hold rq lock. */
  4436. static void
  4437. __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
  4438. {
  4439. BUG_ON(p->se.on_rq);
  4440. p->policy = policy;
  4441. switch (p->policy) {
  4442. case SCHED_NORMAL:
  4443. case SCHED_BATCH:
  4444. case SCHED_IDLE:
  4445. p->sched_class = &fair_sched_class;
  4446. break;
  4447. case SCHED_FIFO:
  4448. case SCHED_RR:
  4449. p->sched_class = &rt_sched_class;
  4450. break;
  4451. }
  4452. p->rt_priority = prio;
  4453. p->normal_prio = normal_prio(p);
  4454. /* we are holding p->pi_lock already */
  4455. p->prio = rt_mutex_getprio(p);
  4456. set_load_weight(p);
  4457. }
  4458. /*
  4459. * check the target process has a UID that matches the current process's
  4460. */
  4461. static bool check_same_owner(struct task_struct *p)
  4462. {
  4463. const struct cred *cred = current_cred(), *pcred;
  4464. bool match;
  4465. rcu_read_lock();
  4466. pcred = __task_cred(p);
  4467. match = (cred->euid == pcred->euid ||
  4468. cred->euid == pcred->uid);
  4469. rcu_read_unlock();
  4470. return match;
  4471. }
  4472. static int __sched_setscheduler(struct task_struct *p, int policy,
  4473. struct sched_param *param, bool user)
  4474. {
  4475. int retval, oldprio, oldpolicy = -1, on_rq, running;
  4476. unsigned long flags;
  4477. const struct sched_class *prev_class = p->sched_class;
  4478. struct rq *rq;
  4479. /* may grab non-irq protected spin_locks */
  4480. BUG_ON(in_interrupt());
  4481. recheck:
  4482. /* double check policy once rq lock held */
  4483. if (policy < 0)
  4484. policy = oldpolicy = p->policy;
  4485. else if (policy != SCHED_FIFO && policy != SCHED_RR &&
  4486. policy != SCHED_NORMAL && policy != SCHED_BATCH &&
  4487. policy != SCHED_IDLE)
  4488. return -EINVAL;
  4489. /*
  4490. * Valid priorities for SCHED_FIFO and SCHED_RR are
  4491. * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
  4492. * SCHED_BATCH and SCHED_IDLE is 0.
  4493. */
  4494. if (param->sched_priority < 0 ||
  4495. (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
  4496. (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
  4497. return -EINVAL;
  4498. if (rt_policy(policy) != (param->sched_priority != 0))
  4499. return -EINVAL;
  4500. /*
  4501. * Allow unprivileged RT tasks to decrease priority:
  4502. */
  4503. if (user && !capable(CAP_SYS_NICE)) {
  4504. if (rt_policy(policy)) {
  4505. unsigned long rlim_rtprio;
  4506. if (!lock_task_sighand(p, &flags))
  4507. return -ESRCH;
  4508. rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
  4509. unlock_task_sighand(p, &flags);
  4510. /* can't set/change the rt policy */
  4511. if (policy != p->policy && !rlim_rtprio)
  4512. return -EPERM;
  4513. /* can't increase priority */
  4514. if (param->sched_priority > p->rt_priority &&
  4515. param->sched_priority > rlim_rtprio)
  4516. return -EPERM;
  4517. }
  4518. /*
  4519. * Like positive nice levels, dont allow tasks to
  4520. * move out of SCHED_IDLE either:
  4521. */
  4522. if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
  4523. return -EPERM;
  4524. /* can't change other user's priorities */
  4525. if (!check_same_owner(p))
  4526. return -EPERM;
  4527. }
  4528. if (user) {
  4529. #ifdef CONFIG_RT_GROUP_SCHED
  4530. /*
  4531. * Do not allow realtime tasks into groups that have no runtime
  4532. * assigned.
  4533. */
  4534. if (rt_bandwidth_enabled() && rt_policy(policy) &&
  4535. task_group(p)->rt_bandwidth.rt_runtime == 0)
  4536. return -EPERM;
  4537. #endif
  4538. retval = security_task_setscheduler(p, policy, param);
  4539. if (retval)
  4540. return retval;
  4541. }
  4542. /*
  4543. * make sure no PI-waiters arrive (or leave) while we are
  4544. * changing the priority of the task:
  4545. */
  4546. spin_lock_irqsave(&p->pi_lock, flags);
  4547. /*
  4548. * To be able to change p->policy safely, the apropriate
  4549. * runqueue lock must be held.
  4550. */
  4551. rq = __task_rq_lock(p);
  4552. /* recheck policy now with rq lock held */
  4553. if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
  4554. policy = oldpolicy = -1;
  4555. __task_rq_unlock(rq);
  4556. spin_unlock_irqrestore(&p->pi_lock, flags);
  4557. goto recheck;
  4558. }
  4559. update_rq_clock(rq);
  4560. on_rq = p->se.on_rq;
  4561. running = task_current(rq, p);
  4562. if (on_rq)
  4563. deactivate_task(rq, p, 0);
  4564. if (running)
  4565. p->sched_class->put_prev_task(rq, p);
  4566. oldprio = p->prio;
  4567. __setscheduler(rq, p, policy, param->sched_priority);
  4568. if (running)
  4569. p->sched_class->set_curr_task(rq);
  4570. if (on_rq) {
  4571. activate_task(rq, p, 0);
  4572. check_class_changed(rq, p, prev_class, oldprio, running);
  4573. }
  4574. __task_rq_unlock(rq);
  4575. spin_unlock_irqrestore(&p->pi_lock, flags);
  4576. rt_mutex_adjust_pi(p);
  4577. return 0;
  4578. }
  4579. /**
  4580. * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
  4581. * @p: the task in question.
  4582. * @policy: new policy.
  4583. * @param: structure containing the new RT priority.
  4584. *
  4585. * NOTE that the task may be already dead.
  4586. */
  4587. int sched_setscheduler(struct task_struct *p, int policy,
  4588. struct sched_param *param)
  4589. {
  4590. return __sched_setscheduler(p, policy, param, true);
  4591. }
  4592. EXPORT_SYMBOL_GPL(sched_setscheduler);
  4593. /**
  4594. * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
  4595. * @p: the task in question.
  4596. * @policy: new policy.
  4597. * @param: structure containing the new RT priority.
  4598. *
  4599. * Just like sched_setscheduler, only don't bother checking if the
  4600. * current context has permission. For example, this is needed in
  4601. * stop_machine(): we create temporary high priority worker threads,
  4602. * but our caller might not have that capability.
  4603. */
  4604. int sched_setscheduler_nocheck(struct task_struct *p, int policy,
  4605. struct sched_param *param)
  4606. {
  4607. return __sched_setscheduler(p, policy, param, false);
  4608. }
  4609. static int
  4610. do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
  4611. {
  4612. struct sched_param lparam;
  4613. struct task_struct *p;
  4614. int retval;
  4615. if (!param || pid < 0)
  4616. return -EINVAL;
  4617. if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
  4618. return -EFAULT;
  4619. rcu_read_lock();
  4620. retval = -ESRCH;
  4621. p = find_process_by_pid(pid);
  4622. if (p != NULL)
  4623. retval = sched_setscheduler(p, policy, &lparam);
  4624. rcu_read_unlock();
  4625. return retval;
  4626. }
  4627. /**
  4628. * sys_sched_setscheduler - set/change the scheduler policy and RT priority
  4629. * @pid: the pid in question.
  4630. * @policy: new policy.
  4631. * @param: structure containing the new RT priority.
  4632. */
  4633. asmlinkage long
  4634. sys_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
  4635. {
  4636. /* negative values for policy are not valid */
  4637. if (policy < 0)
  4638. return -EINVAL;
  4639. return do_sched_setscheduler(pid, policy, param);
  4640. }
  4641. /**
  4642. * sys_sched_setparam - set/change the RT priority of a thread
  4643. * @pid: the pid in question.
  4644. * @param: structure containing the new RT priority.
  4645. */
  4646. asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param)
  4647. {
  4648. return do_sched_setscheduler(pid, -1, param);
  4649. }
  4650. /**
  4651. * sys_sched_getscheduler - get the policy (scheduling class) of a thread
  4652. * @pid: the pid in question.
  4653. */
  4654. asmlinkage long sys_sched_getscheduler(pid_t pid)
  4655. {
  4656. struct task_struct *p;
  4657. int retval;
  4658. if (pid < 0)
  4659. return -EINVAL;
  4660. retval = -ESRCH;
  4661. read_lock(&tasklist_lock);
  4662. p = find_process_by_pid(pid);
  4663. if (p) {
  4664. retval = security_task_getscheduler(p);
  4665. if (!retval)
  4666. retval = p->policy;
  4667. }
  4668. read_unlock(&tasklist_lock);
  4669. return retval;
  4670. }
  4671. /**
  4672. * sys_sched_getscheduler - get the RT priority of a thread
  4673. * @pid: the pid in question.
  4674. * @param: structure containing the RT priority.
  4675. */
  4676. asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param)
  4677. {
  4678. struct sched_param lp;
  4679. struct task_struct *p;
  4680. int retval;
  4681. if (!param || pid < 0)
  4682. return -EINVAL;
  4683. read_lock(&tasklist_lock);
  4684. p = find_process_by_pid(pid);
  4685. retval = -ESRCH;
  4686. if (!p)
  4687. goto out_unlock;
  4688. retval = security_task_getscheduler(p);
  4689. if (retval)
  4690. goto out_unlock;
  4691. lp.sched_priority = p->rt_priority;
  4692. read_unlock(&tasklist_lock);
  4693. /*
  4694. * This one might sleep, we cannot do it with a spinlock held ...
  4695. */
  4696. retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
  4697. return retval;
  4698. out_unlock:
  4699. read_unlock(&tasklist_lock);
  4700. return retval;
  4701. }
  4702. long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
  4703. {
  4704. cpumask_var_t cpus_allowed, new_mask;
  4705. struct task_struct *p;
  4706. int retval;
  4707. get_online_cpus();
  4708. read_lock(&tasklist_lock);
  4709. p = find_process_by_pid(pid);
  4710. if (!p) {
  4711. read_unlock(&tasklist_lock);
  4712. put_online_cpus();
  4713. return -ESRCH;
  4714. }
  4715. /*
  4716. * It is not safe to call set_cpus_allowed with the
  4717. * tasklist_lock held. We will bump the task_struct's
  4718. * usage count and then drop tasklist_lock.
  4719. */
  4720. get_task_struct(p);
  4721. read_unlock(&tasklist_lock);
  4722. if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
  4723. retval = -ENOMEM;
  4724. goto out_put_task;
  4725. }
  4726. if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
  4727. retval = -ENOMEM;
  4728. goto out_free_cpus_allowed;
  4729. }
  4730. retval = -EPERM;
  4731. if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
  4732. goto out_unlock;
  4733. retval = security_task_setscheduler(p, 0, NULL);
  4734. if (retval)
  4735. goto out_unlock;
  4736. cpuset_cpus_allowed(p, cpus_allowed);
  4737. cpumask_and(new_mask, in_mask, cpus_allowed);
  4738. again:
  4739. retval = set_cpus_allowed_ptr(p, new_mask);
  4740. if (!retval) {
  4741. cpuset_cpus_allowed(p, cpus_allowed);
  4742. if (!cpumask_subset(new_mask, cpus_allowed)) {
  4743. /*
  4744. * We must have raced with a concurrent cpuset
  4745. * update. Just reset the cpus_allowed to the
  4746. * cpuset's cpus_allowed
  4747. */
  4748. cpumask_copy(new_mask, cpus_allowed);
  4749. goto again;
  4750. }
  4751. }
  4752. out_unlock:
  4753. free_cpumask_var(new_mask);
  4754. out_free_cpus_allowed:
  4755. free_cpumask_var(cpus_allowed);
  4756. out_put_task:
  4757. put_task_struct(p);
  4758. put_online_cpus();
  4759. return retval;
  4760. }
  4761. static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
  4762. struct cpumask *new_mask)
  4763. {
  4764. if (len < cpumask_size())
  4765. cpumask_clear(new_mask);
  4766. else if (len > cpumask_size())
  4767. len = cpumask_size();
  4768. return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
  4769. }
  4770. /**
  4771. * sys_sched_setaffinity - set the cpu affinity of a process
  4772. * @pid: pid of the process
  4773. * @len: length in bytes of the bitmask pointed to by user_mask_ptr
  4774. * @user_mask_ptr: user-space pointer to the new cpu mask
  4775. */
  4776. asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
  4777. unsigned long __user *user_mask_ptr)
  4778. {
  4779. cpumask_var_t new_mask;
  4780. int retval;
  4781. if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
  4782. return -ENOMEM;
  4783. retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
  4784. if (retval == 0)
  4785. retval = sched_setaffinity(pid, new_mask);
  4786. free_cpumask_var(new_mask);
  4787. return retval;
  4788. }
  4789. long sched_getaffinity(pid_t pid, struct cpumask *mask)
  4790. {
  4791. struct task_struct *p;
  4792. int retval;
  4793. get_online_cpus();
  4794. read_lock(&tasklist_lock);
  4795. retval = -ESRCH;
  4796. p = find_process_by_pid(pid);
  4797. if (!p)
  4798. goto out_unlock;
  4799. retval = security_task_getscheduler(p);
  4800. if (retval)
  4801. goto out_unlock;
  4802. cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
  4803. out_unlock:
  4804. read_unlock(&tasklist_lock);
  4805. put_online_cpus();
  4806. return retval;
  4807. }
  4808. /**
  4809. * sys_sched_getaffinity - get the cpu affinity of a process
  4810. * @pid: pid of the process
  4811. * @len: length in bytes of the bitmask pointed to by user_mask_ptr
  4812. * @user_mask_ptr: user-space pointer to hold the current cpu mask
  4813. */
  4814. asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
  4815. unsigned long __user *user_mask_ptr)
  4816. {
  4817. int ret;
  4818. cpumask_var_t mask;
  4819. if (len < cpumask_size())
  4820. return -EINVAL;
  4821. if (!alloc_cpumask_var(&mask, GFP_KERNEL))
  4822. return -ENOMEM;
  4823. ret = sched_getaffinity(pid, mask);
  4824. if (ret == 0) {
  4825. if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
  4826. ret = -EFAULT;
  4827. else
  4828. ret = cpumask_size();
  4829. }
  4830. free_cpumask_var(mask);
  4831. return ret;
  4832. }
  4833. /**
  4834. * sys_sched_yield - yield the current processor to other threads.
  4835. *
  4836. * This function yields the current CPU to other tasks. If there are no
  4837. * other threads running on this CPU then this function will return.
  4838. */
  4839. asmlinkage long sys_sched_yield(void)
  4840. {
  4841. struct rq *rq = this_rq_lock();
  4842. schedstat_inc(rq, yld_count);
  4843. current->sched_class->yield_task(rq);
  4844. /*
  4845. * Since we are going to call schedule() anyway, there's
  4846. * no need to preempt or enable interrupts:
  4847. */
  4848. __release(rq->lock);
  4849. spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
  4850. _raw_spin_unlock(&rq->lock);
  4851. preempt_enable_no_resched();
  4852. schedule();
  4853. return 0;
  4854. }
  4855. static void __cond_resched(void)
  4856. {
  4857. #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
  4858. __might_sleep(__FILE__, __LINE__);
  4859. #endif
  4860. /*
  4861. * The BKS might be reacquired before we have dropped
  4862. * PREEMPT_ACTIVE, which could trigger a second
  4863. * cond_resched() call.
  4864. */
  4865. do {
  4866. add_preempt_count(PREEMPT_ACTIVE);
  4867. schedule();
  4868. sub_preempt_count(PREEMPT_ACTIVE);
  4869. } while (need_resched());
  4870. }
  4871. int __sched _cond_resched(void)
  4872. {
  4873. if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
  4874. system_state == SYSTEM_RUNNING) {
  4875. __cond_resched();
  4876. return 1;
  4877. }
  4878. return 0;
  4879. }
  4880. EXPORT_SYMBOL(_cond_resched);
  4881. /*
  4882. * cond_resched_lock() - if a reschedule is pending, drop the given lock,
  4883. * call schedule, and on return reacquire the lock.
  4884. *
  4885. * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
  4886. * operations here to prevent schedule() from being called twice (once via
  4887. * spin_unlock(), once by hand).
  4888. */
  4889. int cond_resched_lock(spinlock_t *lock)
  4890. {
  4891. int resched = need_resched() && system_state == SYSTEM_RUNNING;
  4892. int ret = 0;
  4893. if (spin_needbreak(lock) || resched) {
  4894. spin_unlock(lock);
  4895. if (resched && need_resched())
  4896. __cond_resched();
  4897. else
  4898. cpu_relax();
  4899. ret = 1;
  4900. spin_lock(lock);
  4901. }
  4902. return ret;
  4903. }
  4904. EXPORT_SYMBOL(cond_resched_lock);
  4905. int __sched cond_resched_softirq(void)
  4906. {
  4907. BUG_ON(!in_softirq());
  4908. if (need_resched() && system_state == SYSTEM_RUNNING) {
  4909. local_bh_enable();
  4910. __cond_resched();
  4911. local_bh_disable();
  4912. return 1;
  4913. }
  4914. return 0;
  4915. }
  4916. EXPORT_SYMBOL(cond_resched_softirq);
  4917. /**
  4918. * yield - yield the current processor to other threads.
  4919. *
  4920. * This is a shortcut for kernel-space yielding - it marks the
  4921. * thread runnable and calls sys_sched_yield().
  4922. */
  4923. void __sched yield(void)
  4924. {
  4925. set_current_state(TASK_RUNNING);
  4926. sys_sched_yield();
  4927. }
  4928. EXPORT_SYMBOL(yield);
  4929. /*
  4930. * This task is about to go to sleep on IO. Increment rq->nr_iowait so
  4931. * that process accounting knows that this is a task in IO wait state.
  4932. *
  4933. * But don't do that if it is a deliberate, throttling IO wait (this task
  4934. * has set its backing_dev_info: the queue against which it should throttle)
  4935. */
  4936. void __sched io_schedule(void)
  4937. {
  4938. struct rq *rq = &__raw_get_cpu_var(runqueues);
  4939. delayacct_blkio_start();
  4940. atomic_inc(&rq->nr_iowait);
  4941. schedule();
  4942. atomic_dec(&rq->nr_iowait);
  4943. delayacct_blkio_end();
  4944. }
  4945. EXPORT_SYMBOL(io_schedule);
  4946. long __sched io_schedule_timeout(long timeout)
  4947. {
  4948. struct rq *rq = &__raw_get_cpu_var(runqueues);
  4949. long ret;
  4950. delayacct_blkio_start();
  4951. atomic_inc(&rq->nr_iowait);
  4952. ret = schedule_timeout(timeout);
  4953. atomic_dec(&rq->nr_iowait);
  4954. delayacct_blkio_end();
  4955. return ret;
  4956. }
  4957. /**
  4958. * sys_sched_get_priority_max - return maximum RT priority.
  4959. * @policy: scheduling class.
  4960. *
  4961. * this syscall returns the maximum rt_priority that can be used
  4962. * by a given scheduling class.
  4963. */
  4964. asmlinkage long sys_sched_get_priority_max(int policy)
  4965. {
  4966. int ret = -EINVAL;
  4967. switch (policy) {
  4968. case SCHED_FIFO:
  4969. case SCHED_RR:
  4970. ret = MAX_USER_RT_PRIO-1;
  4971. break;
  4972. case SCHED_NORMAL:
  4973. case SCHED_BATCH:
  4974. case SCHED_IDLE:
  4975. ret = 0;
  4976. break;
  4977. }
  4978. return ret;
  4979. }
  4980. /**
  4981. * sys_sched_get_priority_min - return minimum RT priority.
  4982. * @policy: scheduling class.
  4983. *
  4984. * this syscall returns the minimum rt_priority that can be used
  4985. * by a given scheduling class.
  4986. */
  4987. asmlinkage long sys_sched_get_priority_min(int policy)
  4988. {
  4989. int ret = -EINVAL;
  4990. switch (policy) {
  4991. case SCHED_FIFO:
  4992. case SCHED_RR:
  4993. ret = 1;
  4994. break;
  4995. case SCHED_NORMAL:
  4996. case SCHED_BATCH:
  4997. case SCHED_IDLE:
  4998. ret = 0;
  4999. }
  5000. return ret;
  5001. }
  5002. /**
  5003. * sys_sched_rr_get_interval - return the default timeslice of a process.
  5004. * @pid: pid of the process.
  5005. * @interval: userspace pointer to the timeslice value.
  5006. *
  5007. * this syscall writes the default timeslice value of a given process
  5008. * into the user-space timespec buffer. A value of '0' means infinity.
  5009. */
  5010. asmlinkage
  5011. long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval)
  5012. {
  5013. struct task_struct *p;
  5014. unsigned int time_slice;
  5015. int retval;
  5016. struct timespec t;
  5017. if (pid < 0)
  5018. return -EINVAL;
  5019. retval = -ESRCH;
  5020. read_lock(&tasklist_lock);
  5021. p = find_process_by_pid(pid);
  5022. if (!p)
  5023. goto out_unlock;
  5024. retval = security_task_getscheduler(p);
  5025. if (retval)
  5026. goto out_unlock;
  5027. /*
  5028. * Time slice is 0 for SCHED_FIFO tasks and for SCHED_OTHER
  5029. * tasks that are on an otherwise idle runqueue:
  5030. */
  5031. time_slice = 0;
  5032. if (p->policy == SCHED_RR) {
  5033. time_slice = DEF_TIMESLICE;
  5034. } else if (p->policy != SCHED_FIFO) {
  5035. struct sched_entity *se = &p->se;
  5036. unsigned long flags;
  5037. struct rq *rq;
  5038. rq = task_rq_lock(p, &flags);
  5039. if (rq->cfs.load.weight)
  5040. time_slice = NS_TO_JIFFIES(sched_slice(&rq->cfs, se));
  5041. task_rq_unlock(rq, &flags);
  5042. }
  5043. read_unlock(&tasklist_lock);
  5044. jiffies_to_timespec(time_slice, &t);
  5045. retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
  5046. return retval;
  5047. out_unlock:
  5048. read_unlock(&tasklist_lock);
  5049. return retval;
  5050. }
  5051. static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
  5052. void sched_show_task(struct task_struct *p)
  5053. {
  5054. unsigned long free = 0;
  5055. unsigned state;
  5056. state = p->state ? __ffs(p->state) + 1 : 0;
  5057. printk(KERN_INFO "%-13.13s %c", p->comm,
  5058. state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
  5059. #if BITS_PER_LONG == 32
  5060. if (state == TASK_RUNNING)
  5061. printk(KERN_CONT " running ");
  5062. else
  5063. printk(KERN_CONT " %08lx ", thread_saved_pc(p));
  5064. #else
  5065. if (state == TASK_RUNNING)
  5066. printk(KERN_CONT " running task ");
  5067. else
  5068. printk(KERN_CONT " %016lx ", thread_saved_pc(p));
  5069. #endif
  5070. #ifdef CONFIG_DEBUG_STACK_USAGE
  5071. {
  5072. unsigned long *n = end_of_stack(p);
  5073. while (!*n)
  5074. n++;
  5075. free = (unsigned long)n - (unsigned long)end_of_stack(p);
  5076. }
  5077. #endif
  5078. printk(KERN_CONT "%5lu %5d %6d\n", free,
  5079. task_pid_nr(p), task_pid_nr(p->real_parent));
  5080. show_stack(p, NULL);
  5081. }
  5082. void show_state_filter(unsigned long state_filter)
  5083. {
  5084. struct task_struct *g, *p;
  5085. #if BITS_PER_LONG == 32
  5086. printk(KERN_INFO
  5087. " task PC stack pid father\n");
  5088. #else
  5089. printk(KERN_INFO
  5090. " task PC stack pid father\n");
  5091. #endif
  5092. read_lock(&tasklist_lock);
  5093. do_each_thread(g, p) {
  5094. /*
  5095. * reset the NMI-timeout, listing all files on a slow
  5096. * console might take alot of time:
  5097. */
  5098. touch_nmi_watchdog();
  5099. if (!state_filter || (p->state & state_filter))
  5100. sched_show_task(p);
  5101. } while_each_thread(g, p);
  5102. touch_all_softlockup_watchdogs();
  5103. #ifdef CONFIG_SCHED_DEBUG
  5104. sysrq_sched_debug_show();
  5105. #endif
  5106. read_unlock(&tasklist_lock);
  5107. /*
  5108. * Only show locks if all tasks are dumped:
  5109. */
  5110. if (state_filter == -1)
  5111. debug_show_all_locks();
  5112. }
  5113. void __cpuinit init_idle_bootup_task(struct task_struct *idle)
  5114. {
  5115. idle->sched_class = &idle_sched_class;
  5116. }
  5117. /**
  5118. * init_idle - set up an idle thread for a given CPU
  5119. * @idle: task in question
  5120. * @cpu: cpu the idle task belongs to
  5121. *
  5122. * NOTE: this function does not set the idle thread's NEED_RESCHED
  5123. * flag, to make booting more robust.
  5124. */
  5125. void __cpuinit init_idle(struct task_struct *idle, int cpu)
  5126. {
  5127. struct rq *rq = cpu_rq(cpu);
  5128. unsigned long flags;
  5129. spin_lock_irqsave(&rq->lock, flags);
  5130. __sched_fork(idle);
  5131. idle->se.exec_start = sched_clock();
  5132. idle->prio = idle->normal_prio = MAX_PRIO;
  5133. cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
  5134. __set_task_cpu(idle, cpu);
  5135. rq->curr = rq->idle = idle;
  5136. #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
  5137. idle->oncpu = 1;
  5138. #endif
  5139. spin_unlock_irqrestore(&rq->lock, flags);
  5140. /* Set the preempt count _outside_ the spinlocks! */
  5141. #if defined(CONFIG_PREEMPT)
  5142. task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
  5143. #else
  5144. task_thread_info(idle)->preempt_count = 0;
  5145. #endif
  5146. /*
  5147. * The idle tasks have their own, simple scheduling class:
  5148. */
  5149. idle->sched_class = &idle_sched_class;
  5150. ftrace_graph_init_task(idle);
  5151. }
  5152. /*
  5153. * In a system that switches off the HZ timer nohz_cpu_mask
  5154. * indicates which cpus entered this state. This is used
  5155. * in the rcu update to wait only for active cpus. For system
  5156. * which do not switch off the HZ timer nohz_cpu_mask should
  5157. * always be CPU_BITS_NONE.
  5158. */
  5159. cpumask_var_t nohz_cpu_mask;
  5160. /*
  5161. * Increase the granularity value when there are more CPUs,
  5162. * because with more CPUs the 'effective latency' as visible
  5163. * to users decreases. But the relationship is not linear,
  5164. * so pick a second-best guess by going with the log2 of the
  5165. * number of CPUs.
  5166. *
  5167. * This idea comes from the SD scheduler of Con Kolivas:
  5168. */
  5169. static inline void sched_init_granularity(void)
  5170. {
  5171. unsigned int factor = 1 + ilog2(num_online_cpus());
  5172. const unsigned long limit = 200000000;
  5173. sysctl_sched_min_granularity *= factor;
  5174. if (sysctl_sched_min_granularity > limit)
  5175. sysctl_sched_min_granularity = limit;
  5176. sysctl_sched_latency *= factor;
  5177. if (sysctl_sched_latency > limit)
  5178. sysctl_sched_latency = limit;
  5179. sysctl_sched_wakeup_granularity *= factor;
  5180. sysctl_sched_shares_ratelimit *= factor;
  5181. }
  5182. #ifdef CONFIG_SMP
  5183. /*
  5184. * This is how migration works:
  5185. *
  5186. * 1) we queue a struct migration_req structure in the source CPU's
  5187. * runqueue and wake up that CPU's migration thread.
  5188. * 2) we down() the locked semaphore => thread blocks.
  5189. * 3) migration thread wakes up (implicitly it forces the migrated
  5190. * thread off the CPU)
  5191. * 4) it gets the migration request and checks whether the migrated
  5192. * task is still in the wrong runqueue.
  5193. * 5) if it's in the wrong runqueue then the migration thread removes
  5194. * it and puts it into the right queue.
  5195. * 6) migration thread up()s the semaphore.
  5196. * 7) we wake up and the migration is done.
  5197. */
  5198. /*
  5199. * Change a given task's CPU affinity. Migrate the thread to a
  5200. * proper CPU and schedule it away if the CPU it's executing on
  5201. * is removed from the allowed bitmask.
  5202. *
  5203. * NOTE: the caller must have a valid reference to the task, the
  5204. * task must not exit() & deallocate itself prematurely. The
  5205. * call is not atomic; no spinlocks may be held.
  5206. */
  5207. int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
  5208. {
  5209. struct migration_req req;
  5210. unsigned long flags;
  5211. struct rq *rq;
  5212. int ret = 0;
  5213. rq = task_rq_lock(p, &flags);
  5214. if (!cpumask_intersects(new_mask, cpu_online_mask)) {
  5215. ret = -EINVAL;
  5216. goto out;
  5217. }
  5218. if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
  5219. !cpumask_equal(&p->cpus_allowed, new_mask))) {
  5220. ret = -EINVAL;
  5221. goto out;
  5222. }
  5223. if (p->sched_class->set_cpus_allowed)
  5224. p->sched_class->set_cpus_allowed(p, new_mask);
  5225. else {
  5226. cpumask_copy(&p->cpus_allowed, new_mask);
  5227. p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
  5228. }
  5229. /* Can the task run on the task's current CPU? If so, we're done */
  5230. if (cpumask_test_cpu(task_cpu(p), new_mask))
  5231. goto out;
  5232. if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
  5233. /* Need help from migration thread: drop lock and wait. */
  5234. task_rq_unlock(rq, &flags);
  5235. wake_up_process(rq->migration_thread);
  5236. wait_for_completion(&req.done);
  5237. tlb_migrate_finish(p->mm);
  5238. return 0;
  5239. }
  5240. out:
  5241. task_rq_unlock(rq, &flags);
  5242. return ret;
  5243. }
  5244. EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
  5245. /*
  5246. * Move (not current) task off this cpu, onto dest cpu. We're doing
  5247. * this because either it can't run here any more (set_cpus_allowed()
  5248. * away from this CPU, or CPU going down), or because we're
  5249. * attempting to rebalance this task on exec (sched_exec).
  5250. *
  5251. * So we race with normal scheduler movements, but that's OK, as long
  5252. * as the task is no longer on this CPU.
  5253. *
  5254. * Returns non-zero if task was successfully migrated.
  5255. */
  5256. static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
  5257. {
  5258. struct rq *rq_dest, *rq_src;
  5259. int ret = 0, on_rq;
  5260. if (unlikely(!cpu_active(dest_cpu)))
  5261. return ret;
  5262. rq_src = cpu_rq(src_cpu);
  5263. rq_dest = cpu_rq(dest_cpu);
  5264. double_rq_lock(rq_src, rq_dest);
  5265. /* Already moved. */
  5266. if (task_cpu(p) != src_cpu)
  5267. goto done;
  5268. /* Affinity changed (again). */
  5269. if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
  5270. goto fail;
  5271. on_rq = p->se.on_rq;
  5272. if (on_rq)
  5273. deactivate_task(rq_src, p, 0);
  5274. set_task_cpu(p, dest_cpu);
  5275. if (on_rq) {
  5276. activate_task(rq_dest, p, 0);
  5277. check_preempt_curr(rq_dest, p, 0);
  5278. }
  5279. done:
  5280. ret = 1;
  5281. fail:
  5282. double_rq_unlock(rq_src, rq_dest);
  5283. return ret;
  5284. }
  5285. /*
  5286. * migration_thread - this is a highprio system thread that performs
  5287. * thread migration by bumping thread off CPU then 'pushing' onto
  5288. * another runqueue.
  5289. */
  5290. static int migration_thread(void *data)
  5291. {
  5292. int cpu = (long)data;
  5293. struct rq *rq;
  5294. rq = cpu_rq(cpu);
  5295. BUG_ON(rq->migration_thread != current);
  5296. set_current_state(TASK_INTERRUPTIBLE);
  5297. while (!kthread_should_stop()) {
  5298. struct migration_req *req;
  5299. struct list_head *head;
  5300. spin_lock_irq(&rq->lock);
  5301. if (cpu_is_offline(cpu)) {
  5302. spin_unlock_irq(&rq->lock);
  5303. goto wait_to_die;
  5304. }
  5305. if (rq->active_balance) {
  5306. active_load_balance(rq, cpu);
  5307. rq->active_balance = 0;
  5308. }
  5309. head = &rq->migration_queue;
  5310. if (list_empty(head)) {
  5311. spin_unlock_irq(&rq->lock);
  5312. schedule();
  5313. set_current_state(TASK_INTERRUPTIBLE);
  5314. continue;
  5315. }
  5316. req = list_entry(head->next, struct migration_req, list);
  5317. list_del_init(head->next);
  5318. spin_unlock(&rq->lock);
  5319. __migrate_task(req->task, cpu, req->dest_cpu);
  5320. local_irq_enable();
  5321. complete(&req->done);
  5322. }
  5323. __set_current_state(TASK_RUNNING);
  5324. return 0;
  5325. wait_to_die:
  5326. /* Wait for kthread_stop */
  5327. set_current_state(TASK_INTERRUPTIBLE);
  5328. while (!kthread_should_stop()) {
  5329. schedule();
  5330. set_current_state(TASK_INTERRUPTIBLE);
  5331. }
  5332. __set_current_state(TASK_RUNNING);
  5333. return 0;
  5334. }
  5335. #ifdef CONFIG_HOTPLUG_CPU
  5336. static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
  5337. {
  5338. int ret;
  5339. local_irq_disable();
  5340. ret = __migrate_task(p, src_cpu, dest_cpu);
  5341. local_irq_enable();
  5342. return ret;
  5343. }
  5344. /*
  5345. * Figure out where task on dead CPU should go, use force if necessary.
  5346. */
  5347. static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
  5348. {
  5349. int dest_cpu;
  5350. const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
  5351. again:
  5352. /* Look for allowed, online CPU in same node. */
  5353. for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
  5354. if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
  5355. goto move;
  5356. /* Any allowed, online CPU? */
  5357. dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
  5358. if (dest_cpu < nr_cpu_ids)
  5359. goto move;
  5360. /* No more Mr. Nice Guy. */
  5361. if (dest_cpu >= nr_cpu_ids) {
  5362. cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
  5363. dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
  5364. /*
  5365. * Don't tell them about moving exiting tasks or
  5366. * kernel threads (both mm NULL), since they never
  5367. * leave kernel.
  5368. */
  5369. if (p->mm && printk_ratelimit()) {
  5370. printk(KERN_INFO "process %d (%s) no "
  5371. "longer affine to cpu%d\n",
  5372. task_pid_nr(p), p->comm, dead_cpu);
  5373. }
  5374. }
  5375. move:
  5376. /* It can have affinity changed while we were choosing. */
  5377. if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
  5378. goto again;
  5379. }
  5380. /*
  5381. * While a dead CPU has no uninterruptible tasks queued at this point,
  5382. * it might still have a nonzero ->nr_uninterruptible counter, because
  5383. * for performance reasons the counter is not stricly tracking tasks to
  5384. * their home CPUs. So we just add the counter to another CPU's counter,
  5385. * to keep the global sum constant after CPU-down:
  5386. */
  5387. static void migrate_nr_uninterruptible(struct rq *rq_src)
  5388. {
  5389. struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
  5390. unsigned long flags;
  5391. local_irq_save(flags);
  5392. double_rq_lock(rq_src, rq_dest);
  5393. rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
  5394. rq_src->nr_uninterruptible = 0;
  5395. double_rq_unlock(rq_src, rq_dest);
  5396. local_irq_restore(flags);
  5397. }
  5398. /* Run through task list and migrate tasks from the dead cpu. */
  5399. static void migrate_live_tasks(int src_cpu)
  5400. {
  5401. struct task_struct *p, *t;
  5402. read_lock(&tasklist_lock);
  5403. do_each_thread(t, p) {
  5404. if (p == current)
  5405. continue;
  5406. if (task_cpu(p) == src_cpu)
  5407. move_task_off_dead_cpu(src_cpu, p);
  5408. } while_each_thread(t, p);
  5409. read_unlock(&tasklist_lock);
  5410. }
  5411. /*
  5412. * Schedules idle task to be the next runnable task on current CPU.
  5413. * It does so by boosting its priority to highest possible.
  5414. * Used by CPU offline code.
  5415. */
  5416. void sched_idle_next(void)
  5417. {
  5418. int this_cpu = smp_processor_id();
  5419. struct rq *rq = cpu_rq(this_cpu);
  5420. struct task_struct *p = rq->idle;
  5421. unsigned long flags;
  5422. /* cpu has to be offline */
  5423. BUG_ON(cpu_online(this_cpu));
  5424. /*
  5425. * Strictly not necessary since rest of the CPUs are stopped by now
  5426. * and interrupts disabled on the current cpu.
  5427. */
  5428. spin_lock_irqsave(&rq->lock, flags);
  5429. __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
  5430. update_rq_clock(rq);
  5431. activate_task(rq, p, 0);
  5432. spin_unlock_irqrestore(&rq->lock, flags);
  5433. }
  5434. /*
  5435. * Ensures that the idle task is using init_mm right before its cpu goes
  5436. * offline.
  5437. */
  5438. void idle_task_exit(void)
  5439. {
  5440. struct mm_struct *mm = current->active_mm;
  5441. BUG_ON(cpu_online(smp_processor_id()));
  5442. if (mm != &init_mm)
  5443. switch_mm(mm, &init_mm, current);
  5444. mmdrop(mm);
  5445. }
  5446. /* called under rq->lock with disabled interrupts */
  5447. static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
  5448. {
  5449. struct rq *rq = cpu_rq(dead_cpu);
  5450. /* Must be exiting, otherwise would be on tasklist. */
  5451. BUG_ON(!p->exit_state);
  5452. /* Cannot have done final schedule yet: would have vanished. */
  5453. BUG_ON(p->state == TASK_DEAD);
  5454. get_task_struct(p);
  5455. /*
  5456. * Drop lock around migration; if someone else moves it,
  5457. * that's OK. No task can be added to this CPU, so iteration is
  5458. * fine.
  5459. */
  5460. spin_unlock_irq(&rq->lock);
  5461. move_task_off_dead_cpu(dead_cpu, p);
  5462. spin_lock_irq(&rq->lock);
  5463. put_task_struct(p);
  5464. }
  5465. /* release_task() removes task from tasklist, so we won't find dead tasks. */
  5466. static void migrate_dead_tasks(unsigned int dead_cpu)
  5467. {
  5468. struct rq *rq = cpu_rq(dead_cpu);
  5469. struct task_struct *next;
  5470. for ( ; ; ) {
  5471. if (!rq->nr_running)
  5472. break;
  5473. update_rq_clock(rq);
  5474. next = pick_next_task(rq, rq->curr);
  5475. if (!next)
  5476. break;
  5477. next->sched_class->put_prev_task(rq, next);
  5478. migrate_dead(dead_cpu, next);
  5479. }
  5480. }
  5481. #endif /* CONFIG_HOTPLUG_CPU */
  5482. #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
  5483. static struct ctl_table sd_ctl_dir[] = {
  5484. {
  5485. .procname = "sched_domain",
  5486. .mode = 0555,
  5487. },
  5488. {0, },
  5489. };
  5490. static struct ctl_table sd_ctl_root[] = {
  5491. {
  5492. .ctl_name = CTL_KERN,
  5493. .procname = "kernel",
  5494. .mode = 0555,
  5495. .child = sd_ctl_dir,
  5496. },
  5497. {0, },
  5498. };
  5499. static struct ctl_table *sd_alloc_ctl_entry(int n)
  5500. {
  5501. struct ctl_table *entry =
  5502. kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
  5503. return entry;
  5504. }
  5505. static void sd_free_ctl_entry(struct ctl_table **tablep)
  5506. {
  5507. struct ctl_table *entry;
  5508. /*
  5509. * In the intermediate directories, both the child directory and
  5510. * procname are dynamically allocated and could fail but the mode
  5511. * will always be set. In the lowest directory the names are
  5512. * static strings and all have proc handlers.
  5513. */
  5514. for (entry = *tablep; entry->mode; entry++) {
  5515. if (entry->child)
  5516. sd_free_ctl_entry(&entry->child);
  5517. if (entry->proc_handler == NULL)
  5518. kfree(entry->procname);
  5519. }
  5520. kfree(*tablep);
  5521. *tablep = NULL;
  5522. }
  5523. static void
  5524. set_table_entry(struct ctl_table *entry,
  5525. const char *procname, void *data, int maxlen,
  5526. mode_t mode, proc_handler *proc_handler)
  5527. {
  5528. entry->procname = procname;
  5529. entry->data = data;
  5530. entry->maxlen = maxlen;
  5531. entry->mode = mode;
  5532. entry->proc_handler = proc_handler;
  5533. }
  5534. static struct ctl_table *
  5535. sd_alloc_ctl_domain_table(struct sched_domain *sd)
  5536. {
  5537. struct ctl_table *table = sd_alloc_ctl_entry(13);
  5538. if (table == NULL)
  5539. return NULL;
  5540. set_table_entry(&table[0], "min_interval", &sd->min_interval,
  5541. sizeof(long), 0644, proc_doulongvec_minmax);
  5542. set_table_entry(&table[1], "max_interval", &sd->max_interval,
  5543. sizeof(long), 0644, proc_doulongvec_minmax);
  5544. set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
  5545. sizeof(int), 0644, proc_dointvec_minmax);
  5546. set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
  5547. sizeof(int), 0644, proc_dointvec_minmax);
  5548. set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
  5549. sizeof(int), 0644, proc_dointvec_minmax);
  5550. set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
  5551. sizeof(int), 0644, proc_dointvec_minmax);
  5552. set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
  5553. sizeof(int), 0644, proc_dointvec_minmax);
  5554. set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
  5555. sizeof(int), 0644, proc_dointvec_minmax);
  5556. set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
  5557. sizeof(int), 0644, proc_dointvec_minmax);
  5558. set_table_entry(&table[9], "cache_nice_tries",
  5559. &sd->cache_nice_tries,
  5560. sizeof(int), 0644, proc_dointvec_minmax);
  5561. set_table_entry(&table[10], "flags", &sd->flags,
  5562. sizeof(int), 0644, proc_dointvec_minmax);
  5563. set_table_entry(&table[11], "name", sd->name,
  5564. CORENAME_MAX_SIZE, 0444, proc_dostring);
  5565. /* &table[12] is terminator */
  5566. return table;
  5567. }
  5568. static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
  5569. {
  5570. struct ctl_table *entry, *table;
  5571. struct sched_domain *sd;
  5572. int domain_num = 0, i;
  5573. char buf[32];
  5574. for_each_domain(cpu, sd)
  5575. domain_num++;
  5576. entry = table = sd_alloc_ctl_entry(domain_num + 1);
  5577. if (table == NULL)
  5578. return NULL;
  5579. i = 0;
  5580. for_each_domain(cpu, sd) {
  5581. snprintf(buf, 32, "domain%d", i);
  5582. entry->procname = kstrdup(buf, GFP_KERNEL);
  5583. entry->mode = 0555;
  5584. entry->child = sd_alloc_ctl_domain_table(sd);
  5585. entry++;
  5586. i++;
  5587. }
  5588. return table;
  5589. }
  5590. static struct ctl_table_header *sd_sysctl_header;
  5591. static void register_sched_domain_sysctl(void)
  5592. {
  5593. int i, cpu_num = num_online_cpus();
  5594. struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
  5595. char buf[32];
  5596. WARN_ON(sd_ctl_dir[0].child);
  5597. sd_ctl_dir[0].child = entry;
  5598. if (entry == NULL)
  5599. return;
  5600. for_each_online_cpu(i) {
  5601. snprintf(buf, 32, "cpu%d", i);
  5602. entry->procname = kstrdup(buf, GFP_KERNEL);
  5603. entry->mode = 0555;
  5604. entry->child = sd_alloc_ctl_cpu_table(i);
  5605. entry++;
  5606. }
  5607. WARN_ON(sd_sysctl_header);
  5608. sd_sysctl_header = register_sysctl_table(sd_ctl_root);
  5609. }
  5610. /* may be called multiple times per register */
  5611. static void unregister_sched_domain_sysctl(void)
  5612. {
  5613. if (sd_sysctl_header)
  5614. unregister_sysctl_table(sd_sysctl_header);
  5615. sd_sysctl_header = NULL;
  5616. if (sd_ctl_dir[0].child)
  5617. sd_free_ctl_entry(&sd_ctl_dir[0].child);
  5618. }
  5619. #else
  5620. static void register_sched_domain_sysctl(void)
  5621. {
  5622. }
  5623. static void unregister_sched_domain_sysctl(void)
  5624. {
  5625. }
  5626. #endif
  5627. static void set_rq_online(struct rq *rq)
  5628. {
  5629. if (!rq->online) {
  5630. const struct sched_class *class;
  5631. cpumask_set_cpu(rq->cpu, rq->rd->online);
  5632. rq->online = 1;
  5633. for_each_class(class) {
  5634. if (class->rq_online)
  5635. class->rq_online(rq);
  5636. }
  5637. }
  5638. }
  5639. static void set_rq_offline(struct rq *rq)
  5640. {
  5641. if (rq->online) {
  5642. const struct sched_class *class;
  5643. for_each_class(class) {
  5644. if (class->rq_offline)
  5645. class->rq_offline(rq);
  5646. }
  5647. cpumask_clear_cpu(rq->cpu, rq->rd->online);
  5648. rq->online = 0;
  5649. }
  5650. }
  5651. /*
  5652. * migration_call - callback that gets triggered when a CPU is added.
  5653. * Here we can start up the necessary migration thread for the new CPU.
  5654. */
  5655. static int __cpuinit
  5656. migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
  5657. {
  5658. struct task_struct *p;
  5659. int cpu = (long)hcpu;
  5660. unsigned long flags;
  5661. struct rq *rq;
  5662. switch (action) {
  5663. case CPU_UP_PREPARE:
  5664. case CPU_UP_PREPARE_FROZEN:
  5665. p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
  5666. if (IS_ERR(p))
  5667. return NOTIFY_BAD;
  5668. kthread_bind(p, cpu);
  5669. /* Must be high prio: stop_machine expects to yield to it. */
  5670. rq = task_rq_lock(p, &flags);
  5671. __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
  5672. task_rq_unlock(rq, &flags);
  5673. cpu_rq(cpu)->migration_thread = p;
  5674. break;
  5675. case CPU_ONLINE:
  5676. case CPU_ONLINE_FROZEN:
  5677. /* Strictly unnecessary, as first user will wake it. */
  5678. wake_up_process(cpu_rq(cpu)->migration_thread);
  5679. /* Update our root-domain */
  5680. rq = cpu_rq(cpu);
  5681. spin_lock_irqsave(&rq->lock, flags);
  5682. if (rq->rd) {
  5683. BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
  5684. set_rq_online(rq);
  5685. }
  5686. spin_unlock_irqrestore(&rq->lock, flags);
  5687. break;
  5688. #ifdef CONFIG_HOTPLUG_CPU
  5689. case CPU_UP_CANCELED:
  5690. case CPU_UP_CANCELED_FROZEN:
  5691. if (!cpu_rq(cpu)->migration_thread)
  5692. break;
  5693. /* Unbind it from offline cpu so it can run. Fall thru. */
  5694. kthread_bind(cpu_rq(cpu)->migration_thread,
  5695. cpumask_any(cpu_online_mask));
  5696. kthread_stop(cpu_rq(cpu)->migration_thread);
  5697. cpu_rq(cpu)->migration_thread = NULL;
  5698. break;
  5699. case CPU_DEAD:
  5700. case CPU_DEAD_FROZEN:
  5701. cpuset_lock(); /* around calls to cpuset_cpus_allowed_lock() */
  5702. migrate_live_tasks(cpu);
  5703. rq = cpu_rq(cpu);
  5704. kthread_stop(rq->migration_thread);
  5705. rq->migration_thread = NULL;
  5706. /* Idle task back to normal (off runqueue, low prio) */
  5707. spin_lock_irq(&rq->lock);
  5708. update_rq_clock(rq);
  5709. deactivate_task(rq, rq->idle, 0);
  5710. rq->idle->static_prio = MAX_PRIO;
  5711. __setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
  5712. rq->idle->sched_class = &idle_sched_class;
  5713. migrate_dead_tasks(cpu);
  5714. spin_unlock_irq(&rq->lock);
  5715. cpuset_unlock();
  5716. migrate_nr_uninterruptible(rq);
  5717. BUG_ON(rq->nr_running != 0);
  5718. /*
  5719. * No need to migrate the tasks: it was best-effort if
  5720. * they didn't take sched_hotcpu_mutex. Just wake up
  5721. * the requestors.
  5722. */
  5723. spin_lock_irq(&rq->lock);
  5724. while (!list_empty(&rq->migration_queue)) {
  5725. struct migration_req *req;
  5726. req = list_entry(rq->migration_queue.next,
  5727. struct migration_req, list);
  5728. list_del_init(&req->list);
  5729. spin_unlock_irq(&rq->lock);
  5730. complete(&req->done);
  5731. spin_lock_irq(&rq->lock);
  5732. }
  5733. spin_unlock_irq(&rq->lock);
  5734. break;
  5735. case CPU_DYING:
  5736. case CPU_DYING_FROZEN:
  5737. /* Update our root-domain */
  5738. rq = cpu_rq(cpu);
  5739. spin_lock_irqsave(&rq->lock, flags);
  5740. if (rq->rd) {
  5741. BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
  5742. set_rq_offline(rq);
  5743. }
  5744. spin_unlock_irqrestore(&rq->lock, flags);
  5745. break;
  5746. #endif
  5747. }
  5748. return NOTIFY_OK;
  5749. }
  5750. /* Register at highest priority so that task migration (migrate_all_tasks)
  5751. * happens before everything else.
  5752. */
  5753. static struct notifier_block __cpuinitdata migration_notifier = {
  5754. .notifier_call = migration_call,
  5755. .priority = 10
  5756. };
  5757. static int __init migration_init(void)
  5758. {
  5759. void *cpu = (void *)(long)smp_processor_id();
  5760. int err;
  5761. /* Start one for the boot CPU: */
  5762. err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
  5763. BUG_ON(err == NOTIFY_BAD);
  5764. migration_call(&migration_notifier, CPU_ONLINE, cpu);
  5765. register_cpu_notifier(&migration_notifier);
  5766. return err;
  5767. }
  5768. early_initcall(migration_init);
  5769. #endif
  5770. #ifdef CONFIG_SMP
  5771. #ifdef CONFIG_SCHED_DEBUG
  5772. static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
  5773. struct cpumask *groupmask)
  5774. {
  5775. struct sched_group *group = sd->groups;
  5776. char str[256];
  5777. cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
  5778. cpumask_clear(groupmask);
  5779. printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
  5780. if (!(sd->flags & SD_LOAD_BALANCE)) {
  5781. printk("does not load-balance\n");
  5782. if (sd->parent)
  5783. printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
  5784. " has parent");
  5785. return -1;
  5786. }
  5787. printk(KERN_CONT "span %s level %s\n", str, sd->name);
  5788. if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
  5789. printk(KERN_ERR "ERROR: domain->span does not contain "
  5790. "CPU%d\n", cpu);
  5791. }
  5792. if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
  5793. printk(KERN_ERR "ERROR: domain->groups does not contain"
  5794. " CPU%d\n", cpu);
  5795. }
  5796. printk(KERN_DEBUG "%*s groups:", level + 1, "");
  5797. do {
  5798. if (!group) {
  5799. printk("\n");
  5800. printk(KERN_ERR "ERROR: group is NULL\n");
  5801. break;
  5802. }
  5803. if (!group->__cpu_power) {
  5804. printk(KERN_CONT "\n");
  5805. printk(KERN_ERR "ERROR: domain->cpu_power not "
  5806. "set\n");
  5807. break;
  5808. }
  5809. if (!cpumask_weight(sched_group_cpus(group))) {
  5810. printk(KERN_CONT "\n");
  5811. printk(KERN_ERR "ERROR: empty group\n");
  5812. break;
  5813. }
  5814. if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
  5815. printk(KERN_CONT "\n");
  5816. printk(KERN_ERR "ERROR: repeated CPUs\n");
  5817. break;
  5818. }
  5819. cpumask_or(groupmask, groupmask, sched_group_cpus(group));
  5820. cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
  5821. printk(KERN_CONT " %s", str);
  5822. group = group->next;
  5823. } while (group != sd->groups);
  5824. printk(KERN_CONT "\n");
  5825. if (!cpumask_equal(sched_domain_span(sd), groupmask))
  5826. printk(KERN_ERR "ERROR: groups don't span domain->span\n");
  5827. if (sd->parent &&
  5828. !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
  5829. printk(KERN_ERR "ERROR: parent span is not a superset "
  5830. "of domain->span\n");
  5831. return 0;
  5832. }
  5833. static void sched_domain_debug(struct sched_domain *sd, int cpu)
  5834. {
  5835. cpumask_var_t groupmask;
  5836. int level = 0;
  5837. if (!sd) {
  5838. printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
  5839. return;
  5840. }
  5841. printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
  5842. if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
  5843. printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
  5844. return;
  5845. }
  5846. for (;;) {
  5847. if (sched_domain_debug_one(sd, cpu, level, groupmask))
  5848. break;
  5849. level++;
  5850. sd = sd->parent;
  5851. if (!sd)
  5852. break;
  5853. }
  5854. free_cpumask_var(groupmask);
  5855. }
  5856. #else /* !CONFIG_SCHED_DEBUG */
  5857. # define sched_domain_debug(sd, cpu) do { } while (0)
  5858. #endif /* CONFIG_SCHED_DEBUG */
  5859. static int sd_degenerate(struct sched_domain *sd)
  5860. {
  5861. if (cpumask_weight(sched_domain_span(sd)) == 1)
  5862. return 1;
  5863. /* Following flags need at least 2 groups */
  5864. if (sd->flags & (SD_LOAD_BALANCE |
  5865. SD_BALANCE_NEWIDLE |
  5866. SD_BALANCE_FORK |
  5867. SD_BALANCE_EXEC |
  5868. SD_SHARE_CPUPOWER |
  5869. SD_SHARE_PKG_RESOURCES)) {
  5870. if (sd->groups != sd->groups->next)
  5871. return 0;
  5872. }
  5873. /* Following flags don't use groups */
  5874. if (sd->flags & (SD_WAKE_IDLE |
  5875. SD_WAKE_AFFINE |
  5876. SD_WAKE_BALANCE))
  5877. return 0;
  5878. return 1;
  5879. }
  5880. static int
  5881. sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
  5882. {
  5883. unsigned long cflags = sd->flags, pflags = parent->flags;
  5884. if (sd_degenerate(parent))
  5885. return 1;
  5886. if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
  5887. return 0;
  5888. /* Does parent contain flags not in child? */
  5889. /* WAKE_BALANCE is a subset of WAKE_AFFINE */
  5890. if (cflags & SD_WAKE_AFFINE)
  5891. pflags &= ~SD_WAKE_BALANCE;
  5892. /* Flags needing groups don't count if only 1 group in parent */
  5893. if (parent->groups == parent->groups->next) {
  5894. pflags &= ~(SD_LOAD_BALANCE |
  5895. SD_BALANCE_NEWIDLE |
  5896. SD_BALANCE_FORK |
  5897. SD_BALANCE_EXEC |
  5898. SD_SHARE_CPUPOWER |
  5899. SD_SHARE_PKG_RESOURCES);
  5900. if (nr_node_ids == 1)
  5901. pflags &= ~SD_SERIALIZE;
  5902. }
  5903. if (~cflags & pflags)
  5904. return 0;
  5905. return 1;
  5906. }
  5907. static void free_rootdomain(struct root_domain *rd)
  5908. {
  5909. cpupri_cleanup(&rd->cpupri);
  5910. free_cpumask_var(rd->rto_mask);
  5911. free_cpumask_var(rd->online);
  5912. free_cpumask_var(rd->span);
  5913. kfree(rd);
  5914. }
  5915. static void rq_attach_root(struct rq *rq, struct root_domain *rd)
  5916. {
  5917. unsigned long flags;
  5918. spin_lock_irqsave(&rq->lock, flags);
  5919. if (rq->rd) {
  5920. struct root_domain *old_rd = rq->rd;
  5921. if (cpumask_test_cpu(rq->cpu, old_rd->online))
  5922. set_rq_offline(rq);
  5923. cpumask_clear_cpu(rq->cpu, old_rd->span);
  5924. if (atomic_dec_and_test(&old_rd->refcount))
  5925. free_rootdomain(old_rd);
  5926. }
  5927. atomic_inc(&rd->refcount);
  5928. rq->rd = rd;
  5929. cpumask_set_cpu(rq->cpu, rd->span);
  5930. if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
  5931. set_rq_online(rq);
  5932. spin_unlock_irqrestore(&rq->lock, flags);
  5933. }
  5934. static int init_rootdomain(struct root_domain *rd, bool bootmem)
  5935. {
  5936. memset(rd, 0, sizeof(*rd));
  5937. if (bootmem) {
  5938. alloc_bootmem_cpumask_var(&def_root_domain.span);
  5939. alloc_bootmem_cpumask_var(&def_root_domain.online);
  5940. alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
  5941. cpupri_init(&rd->cpupri, true);
  5942. return 0;
  5943. }
  5944. if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
  5945. goto free_rd;
  5946. if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
  5947. goto free_span;
  5948. if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
  5949. goto free_online;
  5950. if (cpupri_init(&rd->cpupri, false) != 0)
  5951. goto free_rto_mask;
  5952. return 0;
  5953. free_rto_mask:
  5954. free_cpumask_var(rd->rto_mask);
  5955. free_online:
  5956. free_cpumask_var(rd->online);
  5957. free_span:
  5958. free_cpumask_var(rd->span);
  5959. free_rd:
  5960. kfree(rd);
  5961. return -ENOMEM;
  5962. }
  5963. static void init_defrootdomain(void)
  5964. {
  5965. init_rootdomain(&def_root_domain, true);
  5966. atomic_set(&def_root_domain.refcount, 1);
  5967. }
  5968. static struct root_domain *alloc_rootdomain(void)
  5969. {
  5970. struct root_domain *rd;
  5971. rd = kmalloc(sizeof(*rd), GFP_KERNEL);
  5972. if (!rd)
  5973. return NULL;
  5974. if (init_rootdomain(rd, false) != 0) {
  5975. kfree(rd);
  5976. return NULL;
  5977. }
  5978. return rd;
  5979. }
  5980. /*
  5981. * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
  5982. * hold the hotplug lock.
  5983. */
  5984. static void
  5985. cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
  5986. {
  5987. struct rq *rq = cpu_rq(cpu);
  5988. struct sched_domain *tmp;
  5989. /* Remove the sched domains which do not contribute to scheduling. */
  5990. for (tmp = sd; tmp; ) {
  5991. struct sched_domain *parent = tmp->parent;
  5992. if (!parent)
  5993. break;
  5994. if (sd_parent_degenerate(tmp, parent)) {
  5995. tmp->parent = parent->parent;
  5996. if (parent->parent)
  5997. parent->parent->child = tmp;
  5998. } else
  5999. tmp = tmp->parent;
  6000. }
  6001. if (sd && sd_degenerate(sd)) {
  6002. sd = sd->parent;
  6003. if (sd)
  6004. sd->child = NULL;
  6005. }
  6006. sched_domain_debug(sd, cpu);
  6007. rq_attach_root(rq, rd);
  6008. rcu_assign_pointer(rq->sd, sd);
  6009. }
  6010. /* cpus with isolated domains */
  6011. static cpumask_var_t cpu_isolated_map;
  6012. /* Setup the mask of cpus configured for isolated domains */
  6013. static int __init isolated_cpu_setup(char *str)
  6014. {
  6015. cpulist_parse(str, cpu_isolated_map);
  6016. return 1;
  6017. }
  6018. __setup("isolcpus=", isolated_cpu_setup);
  6019. /*
  6020. * init_sched_build_groups takes the cpumask we wish to span, and a pointer
  6021. * to a function which identifies what group(along with sched group) a CPU
  6022. * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
  6023. * (due to the fact that we keep track of groups covered with a struct cpumask).
  6024. *
  6025. * init_sched_build_groups will build a circular linked list of the groups
  6026. * covered by the given span, and will set each group's ->cpumask correctly,
  6027. * and ->cpu_power to 0.
  6028. */
  6029. static void
  6030. init_sched_build_groups(const struct cpumask *span,
  6031. const struct cpumask *cpu_map,
  6032. int (*group_fn)(int cpu, const struct cpumask *cpu_map,
  6033. struct sched_group **sg,
  6034. struct cpumask *tmpmask),
  6035. struct cpumask *covered, struct cpumask *tmpmask)
  6036. {
  6037. struct sched_group *first = NULL, *last = NULL;
  6038. int i;
  6039. cpumask_clear(covered);
  6040. for_each_cpu(i, span) {
  6041. struct sched_group *sg;
  6042. int group = group_fn(i, cpu_map, &sg, tmpmask);
  6043. int j;
  6044. if (cpumask_test_cpu(i, covered))
  6045. continue;
  6046. cpumask_clear(sched_group_cpus(sg));
  6047. sg->__cpu_power = 0;
  6048. for_each_cpu(j, span) {
  6049. if (group_fn(j, cpu_map, NULL, tmpmask) != group)
  6050. continue;
  6051. cpumask_set_cpu(j, covered);
  6052. cpumask_set_cpu(j, sched_group_cpus(sg));
  6053. }
  6054. if (!first)
  6055. first = sg;
  6056. if (last)
  6057. last->next = sg;
  6058. last = sg;
  6059. }
  6060. last->next = first;
  6061. }
  6062. #define SD_NODES_PER_DOMAIN 16
  6063. #ifdef CONFIG_NUMA
  6064. /**
  6065. * find_next_best_node - find the next node to include in a sched_domain
  6066. * @node: node whose sched_domain we're building
  6067. * @used_nodes: nodes already in the sched_domain
  6068. *
  6069. * Find the next node to include in a given scheduling domain. Simply
  6070. * finds the closest node not already in the @used_nodes map.
  6071. *
  6072. * Should use nodemask_t.
  6073. */
  6074. static int find_next_best_node(int node, nodemask_t *used_nodes)
  6075. {
  6076. int i, n, val, min_val, best_node = 0;
  6077. min_val = INT_MAX;
  6078. for (i = 0; i < nr_node_ids; i++) {
  6079. /* Start at @node */
  6080. n = (node + i) % nr_node_ids;
  6081. if (!nr_cpus_node(n))
  6082. continue;
  6083. /* Skip already used nodes */
  6084. if (node_isset(n, *used_nodes))
  6085. continue;
  6086. /* Simple min distance search */
  6087. val = node_distance(node, n);
  6088. if (val < min_val) {
  6089. min_val = val;
  6090. best_node = n;
  6091. }
  6092. }
  6093. node_set(best_node, *used_nodes);
  6094. return best_node;
  6095. }
  6096. /**
  6097. * sched_domain_node_span - get a cpumask for a node's sched_domain
  6098. * @node: node whose cpumask we're constructing
  6099. * @span: resulting cpumask
  6100. *
  6101. * Given a node, construct a good cpumask for its sched_domain to span. It
  6102. * should be one that prevents unnecessary balancing, but also spreads tasks
  6103. * out optimally.
  6104. */
  6105. static void sched_domain_node_span(int node, struct cpumask *span)
  6106. {
  6107. nodemask_t used_nodes;
  6108. int i;
  6109. cpumask_clear(span);
  6110. nodes_clear(used_nodes);
  6111. cpumask_or(span, span, cpumask_of_node(node));
  6112. node_set(node, used_nodes);
  6113. for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
  6114. int next_node = find_next_best_node(node, &used_nodes);
  6115. cpumask_or(span, span, cpumask_of_node(next_node));
  6116. }
  6117. }
  6118. #endif /* CONFIG_NUMA */
  6119. int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
  6120. /*
  6121. * The cpus mask in sched_group and sched_domain hangs off the end.
  6122. * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
  6123. * for nr_cpu_ids < CONFIG_NR_CPUS.
  6124. */
  6125. struct static_sched_group {
  6126. struct sched_group sg;
  6127. DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
  6128. };
  6129. struct static_sched_domain {
  6130. struct sched_domain sd;
  6131. DECLARE_BITMAP(span, CONFIG_NR_CPUS);
  6132. };
  6133. /*
  6134. * SMT sched-domains:
  6135. */
  6136. #ifdef CONFIG_SCHED_SMT
  6137. static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
  6138. static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
  6139. static int
  6140. cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
  6141. struct sched_group **sg, struct cpumask *unused)
  6142. {
  6143. if (sg)
  6144. *sg = &per_cpu(sched_group_cpus, cpu).sg;
  6145. return cpu;
  6146. }
  6147. #endif /* CONFIG_SCHED_SMT */
  6148. /*
  6149. * multi-core sched-domains:
  6150. */
  6151. #ifdef CONFIG_SCHED_MC
  6152. static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
  6153. static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
  6154. #endif /* CONFIG_SCHED_MC */
  6155. #if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
  6156. static int
  6157. cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
  6158. struct sched_group **sg, struct cpumask *mask)
  6159. {
  6160. int group;
  6161. cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
  6162. group = cpumask_first(mask);
  6163. if (sg)
  6164. *sg = &per_cpu(sched_group_core, group).sg;
  6165. return group;
  6166. }
  6167. #elif defined(CONFIG_SCHED_MC)
  6168. static int
  6169. cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
  6170. struct sched_group **sg, struct cpumask *unused)
  6171. {
  6172. if (sg)
  6173. *sg = &per_cpu(sched_group_core, cpu).sg;
  6174. return cpu;
  6175. }
  6176. #endif
  6177. static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
  6178. static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
  6179. static int
  6180. cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
  6181. struct sched_group **sg, struct cpumask *mask)
  6182. {
  6183. int group;
  6184. #ifdef CONFIG_SCHED_MC
  6185. cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
  6186. group = cpumask_first(mask);
  6187. #elif defined(CONFIG_SCHED_SMT)
  6188. cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
  6189. group = cpumask_first(mask);
  6190. #else
  6191. group = cpu;
  6192. #endif
  6193. if (sg)
  6194. *sg = &per_cpu(sched_group_phys, group).sg;
  6195. return group;
  6196. }
  6197. #ifdef CONFIG_NUMA
  6198. /*
  6199. * The init_sched_build_groups can't handle what we want to do with node
  6200. * groups, so roll our own. Now each node has its own list of groups which
  6201. * gets dynamically allocated.
  6202. */
  6203. static DEFINE_PER_CPU(struct sched_domain, node_domains);
  6204. static struct sched_group ***sched_group_nodes_bycpu;
  6205. static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
  6206. static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
  6207. static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
  6208. struct sched_group **sg,
  6209. struct cpumask *nodemask)
  6210. {
  6211. int group;
  6212. cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
  6213. group = cpumask_first(nodemask);
  6214. if (sg)
  6215. *sg = &per_cpu(sched_group_allnodes, group).sg;
  6216. return group;
  6217. }
  6218. static void init_numa_sched_groups_power(struct sched_group *group_head)
  6219. {
  6220. struct sched_group *sg = group_head;
  6221. int j;
  6222. if (!sg)
  6223. return;
  6224. do {
  6225. for_each_cpu(j, sched_group_cpus(sg)) {
  6226. struct sched_domain *sd;
  6227. sd = &per_cpu(phys_domains, j).sd;
  6228. if (j != cpumask_first(sched_group_cpus(sd->groups))) {
  6229. /*
  6230. * Only add "power" once for each
  6231. * physical package.
  6232. */
  6233. continue;
  6234. }
  6235. sg_inc_cpu_power(sg, sd->groups->__cpu_power);
  6236. }
  6237. sg = sg->next;
  6238. } while (sg != group_head);
  6239. }
  6240. #endif /* CONFIG_NUMA */
  6241. #ifdef CONFIG_NUMA
  6242. /* Free memory allocated for various sched_group structures */
  6243. static void free_sched_groups(const struct cpumask *cpu_map,
  6244. struct cpumask *nodemask)
  6245. {
  6246. int cpu, i;
  6247. for_each_cpu(cpu, cpu_map) {
  6248. struct sched_group **sched_group_nodes
  6249. = sched_group_nodes_bycpu[cpu];
  6250. if (!sched_group_nodes)
  6251. continue;
  6252. for (i = 0; i < nr_node_ids; i++) {
  6253. struct sched_group *oldsg, *sg = sched_group_nodes[i];
  6254. cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
  6255. if (cpumask_empty(nodemask))
  6256. continue;
  6257. if (sg == NULL)
  6258. continue;
  6259. sg = sg->next;
  6260. next_sg:
  6261. oldsg = sg;
  6262. sg = sg->next;
  6263. kfree(oldsg);
  6264. if (oldsg != sched_group_nodes[i])
  6265. goto next_sg;
  6266. }
  6267. kfree(sched_group_nodes);
  6268. sched_group_nodes_bycpu[cpu] = NULL;
  6269. }
  6270. }
  6271. #else /* !CONFIG_NUMA */
  6272. static void free_sched_groups(const struct cpumask *cpu_map,
  6273. struct cpumask *nodemask)
  6274. {
  6275. }
  6276. #endif /* CONFIG_NUMA */
  6277. /*
  6278. * Initialize sched groups cpu_power.
  6279. *
  6280. * cpu_power indicates the capacity of sched group, which is used while
  6281. * distributing the load between different sched groups in a sched domain.
  6282. * Typically cpu_power for all the groups in a sched domain will be same unless
  6283. * there are asymmetries in the topology. If there are asymmetries, group
  6284. * having more cpu_power will pickup more load compared to the group having
  6285. * less cpu_power.
  6286. *
  6287. * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
  6288. * the maximum number of tasks a group can handle in the presence of other idle
  6289. * or lightly loaded groups in the same sched domain.
  6290. */
  6291. static void init_sched_groups_power(int cpu, struct sched_domain *sd)
  6292. {
  6293. struct sched_domain *child;
  6294. struct sched_group *group;
  6295. WARN_ON(!sd || !sd->groups);
  6296. if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
  6297. return;
  6298. child = sd->child;
  6299. sd->groups->__cpu_power = 0;
  6300. /*
  6301. * For perf policy, if the groups in child domain share resources
  6302. * (for example cores sharing some portions of the cache hierarchy
  6303. * or SMT), then set this domain groups cpu_power such that each group
  6304. * can handle only one task, when there are other idle groups in the
  6305. * same sched domain.
  6306. */
  6307. if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
  6308. (child->flags &
  6309. (SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
  6310. sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
  6311. return;
  6312. }
  6313. /*
  6314. * add cpu_power of each child group to this groups cpu_power
  6315. */
  6316. group = child->groups;
  6317. do {
  6318. sg_inc_cpu_power(sd->groups, group->__cpu_power);
  6319. group = group->next;
  6320. } while (group != child->groups);
  6321. }
  6322. /*
  6323. * Initializers for schedule domains
  6324. * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
  6325. */
  6326. #ifdef CONFIG_SCHED_DEBUG
  6327. # define SD_INIT_NAME(sd, type) sd->name = #type
  6328. #else
  6329. # define SD_INIT_NAME(sd, type) do { } while (0)
  6330. #endif
  6331. #define SD_INIT(sd, type) sd_init_##type(sd)
  6332. #define SD_INIT_FUNC(type) \
  6333. static noinline void sd_init_##type(struct sched_domain *sd) \
  6334. { \
  6335. memset(sd, 0, sizeof(*sd)); \
  6336. *sd = SD_##type##_INIT; \
  6337. sd->level = SD_LV_##type; \
  6338. SD_INIT_NAME(sd, type); \
  6339. }
  6340. SD_INIT_FUNC(CPU)
  6341. #ifdef CONFIG_NUMA
  6342. SD_INIT_FUNC(ALLNODES)
  6343. SD_INIT_FUNC(NODE)
  6344. #endif
  6345. #ifdef CONFIG_SCHED_SMT
  6346. SD_INIT_FUNC(SIBLING)
  6347. #endif
  6348. #ifdef CONFIG_SCHED_MC
  6349. SD_INIT_FUNC(MC)
  6350. #endif
  6351. static int default_relax_domain_level = -1;
  6352. static int __init setup_relax_domain_level(char *str)
  6353. {
  6354. unsigned long val;
  6355. val = simple_strtoul(str, NULL, 0);
  6356. if (val < SD_LV_MAX)
  6357. default_relax_domain_level = val;
  6358. return 1;
  6359. }
  6360. __setup("relax_domain_level=", setup_relax_domain_level);
  6361. static void set_domain_attribute(struct sched_domain *sd,
  6362. struct sched_domain_attr *attr)
  6363. {
  6364. int request;
  6365. if (!attr || attr->relax_domain_level < 0) {
  6366. if (default_relax_domain_level < 0)
  6367. return;
  6368. else
  6369. request = default_relax_domain_level;
  6370. } else
  6371. request = attr->relax_domain_level;
  6372. if (request < sd->level) {
  6373. /* turn off idle balance on this domain */
  6374. sd->flags &= ~(SD_WAKE_IDLE|SD_BALANCE_NEWIDLE);
  6375. } else {
  6376. /* turn on idle balance on this domain */
  6377. sd->flags |= (SD_WAKE_IDLE_FAR|SD_BALANCE_NEWIDLE);
  6378. }
  6379. }
  6380. /*
  6381. * Build sched domains for a given set of cpus and attach the sched domains
  6382. * to the individual cpus
  6383. */
  6384. static int __build_sched_domains(const struct cpumask *cpu_map,
  6385. struct sched_domain_attr *attr)
  6386. {
  6387. int i, err = -ENOMEM;
  6388. struct root_domain *rd;
  6389. cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
  6390. tmpmask;
  6391. #ifdef CONFIG_NUMA
  6392. cpumask_var_t domainspan, covered, notcovered;
  6393. struct sched_group **sched_group_nodes = NULL;
  6394. int sd_allnodes = 0;
  6395. if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
  6396. goto out;
  6397. if (!alloc_cpumask_var(&covered, GFP_KERNEL))
  6398. goto free_domainspan;
  6399. if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
  6400. goto free_covered;
  6401. #endif
  6402. if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
  6403. goto free_notcovered;
  6404. if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
  6405. goto free_nodemask;
  6406. if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
  6407. goto free_this_sibling_map;
  6408. if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
  6409. goto free_this_core_map;
  6410. if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
  6411. goto free_send_covered;
  6412. #ifdef CONFIG_NUMA
  6413. /*
  6414. * Allocate the per-node list of sched groups
  6415. */
  6416. sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *),
  6417. GFP_KERNEL);
  6418. if (!sched_group_nodes) {
  6419. printk(KERN_WARNING "Can not alloc sched group node list\n");
  6420. goto free_tmpmask;
  6421. }
  6422. #endif
  6423. rd = alloc_rootdomain();
  6424. if (!rd) {
  6425. printk(KERN_WARNING "Cannot alloc root domain\n");
  6426. goto free_sched_groups;
  6427. }
  6428. #ifdef CONFIG_NUMA
  6429. sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
  6430. #endif
  6431. /*
  6432. * Set up domains for cpus specified by the cpu_map.
  6433. */
  6434. for_each_cpu(i, cpu_map) {
  6435. struct sched_domain *sd = NULL, *p;
  6436. cpumask_and(nodemask, cpumask_of_node(cpu_to_node(i)), cpu_map);
  6437. #ifdef CONFIG_NUMA
  6438. if (cpumask_weight(cpu_map) >
  6439. SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
  6440. sd = &per_cpu(allnodes_domains, i);
  6441. SD_INIT(sd, ALLNODES);
  6442. set_domain_attribute(sd, attr);
  6443. cpumask_copy(sched_domain_span(sd), cpu_map);
  6444. cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
  6445. p = sd;
  6446. sd_allnodes = 1;
  6447. } else
  6448. p = NULL;
  6449. sd = &per_cpu(node_domains, i);
  6450. SD_INIT(sd, NODE);
  6451. set_domain_attribute(sd, attr);
  6452. sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
  6453. sd->parent = p;
  6454. if (p)
  6455. p->child = sd;
  6456. cpumask_and(sched_domain_span(sd),
  6457. sched_domain_span(sd), cpu_map);
  6458. #endif
  6459. p = sd;
  6460. sd = &per_cpu(phys_domains, i).sd;
  6461. SD_INIT(sd, CPU);
  6462. set_domain_attribute(sd, attr);
  6463. cpumask_copy(sched_domain_span(sd), nodemask);
  6464. sd->parent = p;
  6465. if (p)
  6466. p->child = sd;
  6467. cpu_to_phys_group(i, cpu_map, &sd->groups, tmpmask);
  6468. #ifdef CONFIG_SCHED_MC
  6469. p = sd;
  6470. sd = &per_cpu(core_domains, i).sd;
  6471. SD_INIT(sd, MC);
  6472. set_domain_attribute(sd, attr);
  6473. cpumask_and(sched_domain_span(sd), cpu_map,
  6474. cpu_coregroup_mask(i));
  6475. sd->parent = p;
  6476. p->child = sd;
  6477. cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
  6478. #endif
  6479. #ifdef CONFIG_SCHED_SMT
  6480. p = sd;
  6481. sd = &per_cpu(cpu_domains, i).sd;
  6482. SD_INIT(sd, SIBLING);
  6483. set_domain_attribute(sd, attr);
  6484. cpumask_and(sched_domain_span(sd),
  6485. &per_cpu(cpu_sibling_map, i), cpu_map);
  6486. sd->parent = p;
  6487. p->child = sd;
  6488. cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
  6489. #endif
  6490. }
  6491. #ifdef CONFIG_SCHED_SMT
  6492. /* Set up CPU (sibling) groups */
  6493. for_each_cpu(i, cpu_map) {
  6494. cpumask_and(this_sibling_map,
  6495. &per_cpu(cpu_sibling_map, i), cpu_map);
  6496. if (i != cpumask_first(this_sibling_map))
  6497. continue;
  6498. init_sched_build_groups(this_sibling_map, cpu_map,
  6499. &cpu_to_cpu_group,
  6500. send_covered, tmpmask);
  6501. }
  6502. #endif
  6503. #ifdef CONFIG_SCHED_MC
  6504. /* Set up multi-core groups */
  6505. for_each_cpu(i, cpu_map) {
  6506. cpumask_and(this_core_map, cpu_coregroup_mask(i), cpu_map);
  6507. if (i != cpumask_first(this_core_map))
  6508. continue;
  6509. init_sched_build_groups(this_core_map, cpu_map,
  6510. &cpu_to_core_group,
  6511. send_covered, tmpmask);
  6512. }
  6513. #endif
  6514. /* Set up physical groups */
  6515. for (i = 0; i < nr_node_ids; i++) {
  6516. cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
  6517. if (cpumask_empty(nodemask))
  6518. continue;
  6519. init_sched_build_groups(nodemask, cpu_map,
  6520. &cpu_to_phys_group,
  6521. send_covered, tmpmask);
  6522. }
  6523. #ifdef CONFIG_NUMA
  6524. /* Set up node groups */
  6525. if (sd_allnodes) {
  6526. init_sched_build_groups(cpu_map, cpu_map,
  6527. &cpu_to_allnodes_group,
  6528. send_covered, tmpmask);
  6529. }
  6530. for (i = 0; i < nr_node_ids; i++) {
  6531. /* Set up node groups */
  6532. struct sched_group *sg, *prev;
  6533. int j;
  6534. cpumask_clear(covered);
  6535. cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
  6536. if (cpumask_empty(nodemask)) {
  6537. sched_group_nodes[i] = NULL;
  6538. continue;
  6539. }
  6540. sched_domain_node_span(i, domainspan);
  6541. cpumask_and(domainspan, domainspan, cpu_map);
  6542. sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
  6543. GFP_KERNEL, i);
  6544. if (!sg) {
  6545. printk(KERN_WARNING "Can not alloc domain group for "
  6546. "node %d\n", i);
  6547. goto error;
  6548. }
  6549. sched_group_nodes[i] = sg;
  6550. for_each_cpu(j, nodemask) {
  6551. struct sched_domain *sd;
  6552. sd = &per_cpu(node_domains, j);
  6553. sd->groups = sg;
  6554. }
  6555. sg->__cpu_power = 0;
  6556. cpumask_copy(sched_group_cpus(sg), nodemask);
  6557. sg->next = sg;
  6558. cpumask_or(covered, covered, nodemask);
  6559. prev = sg;
  6560. for (j = 0; j < nr_node_ids; j++) {
  6561. int n = (i + j) % nr_node_ids;
  6562. cpumask_complement(notcovered, covered);
  6563. cpumask_and(tmpmask, notcovered, cpu_map);
  6564. cpumask_and(tmpmask, tmpmask, domainspan);
  6565. if (cpumask_empty(tmpmask))
  6566. break;
  6567. cpumask_and(tmpmask, tmpmask, cpumask_of_node(n));
  6568. if (cpumask_empty(tmpmask))
  6569. continue;
  6570. sg = kmalloc_node(sizeof(struct sched_group) +
  6571. cpumask_size(),
  6572. GFP_KERNEL, i);
  6573. if (!sg) {
  6574. printk(KERN_WARNING
  6575. "Can not alloc domain group for node %d\n", j);
  6576. goto error;
  6577. }
  6578. sg->__cpu_power = 0;
  6579. cpumask_copy(sched_group_cpus(sg), tmpmask);
  6580. sg->next = prev->next;
  6581. cpumask_or(covered, covered, tmpmask);
  6582. prev->next = sg;
  6583. prev = sg;
  6584. }
  6585. }
  6586. #endif
  6587. /* Calculate CPU power for physical packages and nodes */
  6588. #ifdef CONFIG_SCHED_SMT
  6589. for_each_cpu(i, cpu_map) {
  6590. struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
  6591. init_sched_groups_power(i, sd);
  6592. }
  6593. #endif
  6594. #ifdef CONFIG_SCHED_MC
  6595. for_each_cpu(i, cpu_map) {
  6596. struct sched_domain *sd = &per_cpu(core_domains, i).sd;
  6597. init_sched_groups_power(i, sd);
  6598. }
  6599. #endif
  6600. for_each_cpu(i, cpu_map) {
  6601. struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
  6602. init_sched_groups_power(i, sd);
  6603. }
  6604. #ifdef CONFIG_NUMA
  6605. for (i = 0; i < nr_node_ids; i++)
  6606. init_numa_sched_groups_power(sched_group_nodes[i]);
  6607. if (sd_allnodes) {
  6608. struct sched_group *sg;
  6609. cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
  6610. tmpmask);
  6611. init_numa_sched_groups_power(sg);
  6612. }
  6613. #endif
  6614. /* Attach the domains */
  6615. for_each_cpu(i, cpu_map) {
  6616. struct sched_domain *sd;
  6617. #ifdef CONFIG_SCHED_SMT
  6618. sd = &per_cpu(cpu_domains, i).sd;
  6619. #elif defined(CONFIG_SCHED_MC)
  6620. sd = &per_cpu(core_domains, i).sd;
  6621. #else
  6622. sd = &per_cpu(phys_domains, i).sd;
  6623. #endif
  6624. cpu_attach_domain(sd, rd, i);
  6625. }
  6626. err = 0;
  6627. free_tmpmask:
  6628. free_cpumask_var(tmpmask);
  6629. free_send_covered:
  6630. free_cpumask_var(send_covered);
  6631. free_this_core_map:
  6632. free_cpumask_var(this_core_map);
  6633. free_this_sibling_map:
  6634. free_cpumask_var(this_sibling_map);
  6635. free_nodemask:
  6636. free_cpumask_var(nodemask);
  6637. free_notcovered:
  6638. #ifdef CONFIG_NUMA
  6639. free_cpumask_var(notcovered);
  6640. free_covered:
  6641. free_cpumask_var(covered);
  6642. free_domainspan:
  6643. free_cpumask_var(domainspan);
  6644. out:
  6645. #endif
  6646. return err;
  6647. free_sched_groups:
  6648. #ifdef CONFIG_NUMA
  6649. kfree(sched_group_nodes);
  6650. #endif
  6651. goto free_tmpmask;
  6652. #ifdef CONFIG_NUMA
  6653. error:
  6654. free_sched_groups(cpu_map, tmpmask);
  6655. free_rootdomain(rd);
  6656. goto free_tmpmask;
  6657. #endif
  6658. }
  6659. static int build_sched_domains(const struct cpumask *cpu_map)
  6660. {
  6661. return __build_sched_domains(cpu_map, NULL);
  6662. }
  6663. static struct cpumask *doms_cur; /* current sched domains */
  6664. static int ndoms_cur; /* number of sched domains in 'doms_cur' */
  6665. static struct sched_domain_attr *dattr_cur;
  6666. /* attribues of custom domains in 'doms_cur' */
  6667. /*
  6668. * Special case: If a kmalloc of a doms_cur partition (array of
  6669. * cpumask) fails, then fallback to a single sched domain,
  6670. * as determined by the single cpumask fallback_doms.
  6671. */
  6672. static cpumask_var_t fallback_doms;
  6673. /*
  6674. * arch_update_cpu_topology lets virtualized architectures update the
  6675. * cpu core maps. It is supposed to return 1 if the topology changed
  6676. * or 0 if it stayed the same.
  6677. */
  6678. int __attribute__((weak)) arch_update_cpu_topology(void)
  6679. {
  6680. return 0;
  6681. }
  6682. /*
  6683. * Set up scheduler domains and groups. Callers must hold the hotplug lock.
  6684. * For now this just excludes isolated cpus, but could be used to
  6685. * exclude other special cases in the future.
  6686. */
  6687. static int arch_init_sched_domains(const struct cpumask *cpu_map)
  6688. {
  6689. int err;
  6690. arch_update_cpu_topology();
  6691. ndoms_cur = 1;
  6692. doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
  6693. if (!doms_cur)
  6694. doms_cur = fallback_doms;
  6695. cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
  6696. dattr_cur = NULL;
  6697. err = build_sched_domains(doms_cur);
  6698. register_sched_domain_sysctl();
  6699. return err;
  6700. }
  6701. static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
  6702. struct cpumask *tmpmask)
  6703. {
  6704. free_sched_groups(cpu_map, tmpmask);
  6705. }
  6706. /*
  6707. * Detach sched domains from a group of cpus specified in cpu_map
  6708. * These cpus will now be attached to the NULL domain
  6709. */
  6710. static void detach_destroy_domains(const struct cpumask *cpu_map)
  6711. {
  6712. /* Save because hotplug lock held. */
  6713. static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
  6714. int i;
  6715. for_each_cpu(i, cpu_map)
  6716. cpu_attach_domain(NULL, &def_root_domain, i);
  6717. synchronize_sched();
  6718. arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
  6719. }
  6720. /* handle null as "default" */
  6721. static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
  6722. struct sched_domain_attr *new, int idx_new)
  6723. {
  6724. struct sched_domain_attr tmp;
  6725. /* fast path */
  6726. if (!new && !cur)
  6727. return 1;
  6728. tmp = SD_ATTR_INIT;
  6729. return !memcmp(cur ? (cur + idx_cur) : &tmp,
  6730. new ? (new + idx_new) : &tmp,
  6731. sizeof(struct sched_domain_attr));
  6732. }
  6733. /*
  6734. * Partition sched domains as specified by the 'ndoms_new'
  6735. * cpumasks in the array doms_new[] of cpumasks. This compares
  6736. * doms_new[] to the current sched domain partitioning, doms_cur[].
  6737. * It destroys each deleted domain and builds each new domain.
  6738. *
  6739. * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
  6740. * The masks don't intersect (don't overlap.) We should setup one
  6741. * sched domain for each mask. CPUs not in any of the cpumasks will
  6742. * not be load balanced. If the same cpumask appears both in the
  6743. * current 'doms_cur' domains and in the new 'doms_new', we can leave
  6744. * it as it is.
  6745. *
  6746. * The passed in 'doms_new' should be kmalloc'd. This routine takes
  6747. * ownership of it and will kfree it when done with it. If the caller
  6748. * failed the kmalloc call, then it can pass in doms_new == NULL &&
  6749. * ndoms_new == 1, and partition_sched_domains() will fallback to
  6750. * the single partition 'fallback_doms', it also forces the domains
  6751. * to be rebuilt.
  6752. *
  6753. * If doms_new == NULL it will be replaced with cpu_online_mask.
  6754. * ndoms_new == 0 is a special case for destroying existing domains,
  6755. * and it will not create the default domain.
  6756. *
  6757. * Call with hotplug lock held
  6758. */
  6759. /* FIXME: Change to struct cpumask *doms_new[] */
  6760. void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
  6761. struct sched_domain_attr *dattr_new)
  6762. {
  6763. int i, j, n;
  6764. int new_topology;
  6765. mutex_lock(&sched_domains_mutex);
  6766. /* always unregister in case we don't destroy any domains */
  6767. unregister_sched_domain_sysctl();
  6768. /* Let architecture update cpu core mappings. */
  6769. new_topology = arch_update_cpu_topology();
  6770. n = doms_new ? ndoms_new : 0;
  6771. /* Destroy deleted domains */
  6772. for (i = 0; i < ndoms_cur; i++) {
  6773. for (j = 0; j < n && !new_topology; j++) {
  6774. if (cpumask_equal(&doms_cur[i], &doms_new[j])
  6775. && dattrs_equal(dattr_cur, i, dattr_new, j))
  6776. goto match1;
  6777. }
  6778. /* no match - a current sched domain not in new doms_new[] */
  6779. detach_destroy_domains(doms_cur + i);
  6780. match1:
  6781. ;
  6782. }
  6783. if (doms_new == NULL) {
  6784. ndoms_cur = 0;
  6785. doms_new = fallback_doms;
  6786. cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
  6787. WARN_ON_ONCE(dattr_new);
  6788. }
  6789. /* Build new domains */
  6790. for (i = 0; i < ndoms_new; i++) {
  6791. for (j = 0; j < ndoms_cur && !new_topology; j++) {
  6792. if (cpumask_equal(&doms_new[i], &doms_cur[j])
  6793. && dattrs_equal(dattr_new, i, dattr_cur, j))
  6794. goto match2;
  6795. }
  6796. /* no match - add a new doms_new */
  6797. __build_sched_domains(doms_new + i,
  6798. dattr_new ? dattr_new + i : NULL);
  6799. match2:
  6800. ;
  6801. }
  6802. /* Remember the new sched domains */
  6803. if (doms_cur != fallback_doms)
  6804. kfree(doms_cur);
  6805. kfree(dattr_cur); /* kfree(NULL) is safe */
  6806. doms_cur = doms_new;
  6807. dattr_cur = dattr_new;
  6808. ndoms_cur = ndoms_new;
  6809. register_sched_domain_sysctl();
  6810. mutex_unlock(&sched_domains_mutex);
  6811. }
  6812. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  6813. int arch_reinit_sched_domains(void)
  6814. {
  6815. get_online_cpus();
  6816. /* Destroy domains first to force the rebuild */
  6817. partition_sched_domains(0, NULL, NULL);
  6818. rebuild_sched_domains();
  6819. put_online_cpus();
  6820. return 0;
  6821. }
  6822. static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
  6823. {
  6824. int ret;
  6825. unsigned int level = 0;
  6826. if (sscanf(buf, "%u", &level) != 1)
  6827. return -EINVAL;
  6828. /*
  6829. * level is always be positive so don't check for
  6830. * level < POWERSAVINGS_BALANCE_NONE which is 0
  6831. * What happens on 0 or 1 byte write,
  6832. * need to check for count as well?
  6833. */
  6834. if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
  6835. return -EINVAL;
  6836. if (smt)
  6837. sched_smt_power_savings = level;
  6838. else
  6839. sched_mc_power_savings = level;
  6840. ret = arch_reinit_sched_domains();
  6841. return ret ? ret : count;
  6842. }
  6843. #ifdef CONFIG_SCHED_MC
  6844. static ssize_t sched_mc_power_savings_show(struct sysdev_class *class,
  6845. char *page)
  6846. {
  6847. return sprintf(page, "%u\n", sched_mc_power_savings);
  6848. }
  6849. static ssize_t sched_mc_power_savings_store(struct sysdev_class *class,
  6850. const char *buf, size_t count)
  6851. {
  6852. return sched_power_savings_store(buf, count, 0);
  6853. }
  6854. static SYSDEV_CLASS_ATTR(sched_mc_power_savings, 0644,
  6855. sched_mc_power_savings_show,
  6856. sched_mc_power_savings_store);
  6857. #endif
  6858. #ifdef CONFIG_SCHED_SMT
  6859. static ssize_t sched_smt_power_savings_show(struct sysdev_class *dev,
  6860. char *page)
  6861. {
  6862. return sprintf(page, "%u\n", sched_smt_power_savings);
  6863. }
  6864. static ssize_t sched_smt_power_savings_store(struct sysdev_class *dev,
  6865. const char *buf, size_t count)
  6866. {
  6867. return sched_power_savings_store(buf, count, 1);
  6868. }
  6869. static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
  6870. sched_smt_power_savings_show,
  6871. sched_smt_power_savings_store);
  6872. #endif
  6873. int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
  6874. {
  6875. int err = 0;
  6876. #ifdef CONFIG_SCHED_SMT
  6877. if (smt_capable())
  6878. err = sysfs_create_file(&cls->kset.kobj,
  6879. &attr_sched_smt_power_savings.attr);
  6880. #endif
  6881. #ifdef CONFIG_SCHED_MC
  6882. if (!err && mc_capable())
  6883. err = sysfs_create_file(&cls->kset.kobj,
  6884. &attr_sched_mc_power_savings.attr);
  6885. #endif
  6886. return err;
  6887. }
  6888. #endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
  6889. #ifndef CONFIG_CPUSETS
  6890. /*
  6891. * Add online and remove offline CPUs from the scheduler domains.
  6892. * When cpusets are enabled they take over this function.
  6893. */
  6894. static int update_sched_domains(struct notifier_block *nfb,
  6895. unsigned long action, void *hcpu)
  6896. {
  6897. switch (action) {
  6898. case CPU_ONLINE:
  6899. case CPU_ONLINE_FROZEN:
  6900. case CPU_DEAD:
  6901. case CPU_DEAD_FROZEN:
  6902. partition_sched_domains(1, NULL, NULL);
  6903. return NOTIFY_OK;
  6904. default:
  6905. return NOTIFY_DONE;
  6906. }
  6907. }
  6908. #endif
  6909. static int update_runtime(struct notifier_block *nfb,
  6910. unsigned long action, void *hcpu)
  6911. {
  6912. int cpu = (int)(long)hcpu;
  6913. switch (action) {
  6914. case CPU_DOWN_PREPARE:
  6915. case CPU_DOWN_PREPARE_FROZEN:
  6916. disable_runtime(cpu_rq(cpu));
  6917. return NOTIFY_OK;
  6918. case CPU_DOWN_FAILED:
  6919. case CPU_DOWN_FAILED_FROZEN:
  6920. case CPU_ONLINE:
  6921. case CPU_ONLINE_FROZEN:
  6922. enable_runtime(cpu_rq(cpu));
  6923. return NOTIFY_OK;
  6924. default:
  6925. return NOTIFY_DONE;
  6926. }
  6927. }
  6928. void __init sched_init_smp(void)
  6929. {
  6930. cpumask_var_t non_isolated_cpus;
  6931. alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
  6932. #if defined(CONFIG_NUMA)
  6933. sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
  6934. GFP_KERNEL);
  6935. BUG_ON(sched_group_nodes_bycpu == NULL);
  6936. #endif
  6937. get_online_cpus();
  6938. mutex_lock(&sched_domains_mutex);
  6939. arch_init_sched_domains(cpu_online_mask);
  6940. cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
  6941. if (cpumask_empty(non_isolated_cpus))
  6942. cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
  6943. mutex_unlock(&sched_domains_mutex);
  6944. put_online_cpus();
  6945. #ifndef CONFIG_CPUSETS
  6946. /* XXX: Theoretical race here - CPU may be hotplugged now */
  6947. hotcpu_notifier(update_sched_domains, 0);
  6948. #endif
  6949. /* RT runtime code needs to handle some hotplug events */
  6950. hotcpu_notifier(update_runtime, 0);
  6951. init_hrtick();
  6952. /* Move init over to a non-isolated CPU */
  6953. if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
  6954. BUG();
  6955. sched_init_granularity();
  6956. free_cpumask_var(non_isolated_cpus);
  6957. alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
  6958. init_sched_rt_class();
  6959. }
  6960. #else
  6961. void __init sched_init_smp(void)
  6962. {
  6963. sched_init_granularity();
  6964. }
  6965. #endif /* CONFIG_SMP */
  6966. int in_sched_functions(unsigned long addr)
  6967. {
  6968. return in_lock_functions(addr) ||
  6969. (addr >= (unsigned long)__sched_text_start
  6970. && addr < (unsigned long)__sched_text_end);
  6971. }
  6972. static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
  6973. {
  6974. cfs_rq->tasks_timeline = RB_ROOT;
  6975. INIT_LIST_HEAD(&cfs_rq->tasks);
  6976. #ifdef CONFIG_FAIR_GROUP_SCHED
  6977. cfs_rq->rq = rq;
  6978. #endif
  6979. cfs_rq->min_vruntime = (u64)(-(1LL << 20));
  6980. }
  6981. static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
  6982. {
  6983. struct rt_prio_array *array;
  6984. int i;
  6985. array = &rt_rq->active;
  6986. for (i = 0; i < MAX_RT_PRIO; i++) {
  6987. INIT_LIST_HEAD(array->queue + i);
  6988. __clear_bit(i, array->bitmap);
  6989. }
  6990. /* delimiter for bitsearch: */
  6991. __set_bit(MAX_RT_PRIO, array->bitmap);
  6992. #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
  6993. rt_rq->highest_prio = MAX_RT_PRIO;
  6994. #endif
  6995. #ifdef CONFIG_SMP
  6996. rt_rq->rt_nr_migratory = 0;
  6997. rt_rq->overloaded = 0;
  6998. #endif
  6999. rt_rq->rt_time = 0;
  7000. rt_rq->rt_throttled = 0;
  7001. rt_rq->rt_runtime = 0;
  7002. spin_lock_init(&rt_rq->rt_runtime_lock);
  7003. #ifdef CONFIG_RT_GROUP_SCHED
  7004. rt_rq->rt_nr_boosted = 0;
  7005. rt_rq->rq = rq;
  7006. #endif
  7007. }
  7008. #ifdef CONFIG_FAIR_GROUP_SCHED
  7009. static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
  7010. struct sched_entity *se, int cpu, int add,
  7011. struct sched_entity *parent)
  7012. {
  7013. struct rq *rq = cpu_rq(cpu);
  7014. tg->cfs_rq[cpu] = cfs_rq;
  7015. init_cfs_rq(cfs_rq, rq);
  7016. cfs_rq->tg = tg;
  7017. if (add)
  7018. list_add(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
  7019. tg->se[cpu] = se;
  7020. /* se could be NULL for init_task_group */
  7021. if (!se)
  7022. return;
  7023. if (!parent)
  7024. se->cfs_rq = &rq->cfs;
  7025. else
  7026. se->cfs_rq = parent->my_q;
  7027. se->my_q = cfs_rq;
  7028. se->load.weight = tg->shares;
  7029. se->load.inv_weight = 0;
  7030. se->parent = parent;
  7031. }
  7032. #endif
  7033. #ifdef CONFIG_RT_GROUP_SCHED
  7034. static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
  7035. struct sched_rt_entity *rt_se, int cpu, int add,
  7036. struct sched_rt_entity *parent)
  7037. {
  7038. struct rq *rq = cpu_rq(cpu);
  7039. tg->rt_rq[cpu] = rt_rq;
  7040. init_rt_rq(rt_rq, rq);
  7041. rt_rq->tg = tg;
  7042. rt_rq->rt_se = rt_se;
  7043. rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
  7044. if (add)
  7045. list_add(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);
  7046. tg->rt_se[cpu] = rt_se;
  7047. if (!rt_se)
  7048. return;
  7049. if (!parent)
  7050. rt_se->rt_rq = &rq->rt;
  7051. else
  7052. rt_se->rt_rq = parent->my_q;
  7053. rt_se->my_q = rt_rq;
  7054. rt_se->parent = parent;
  7055. INIT_LIST_HEAD(&rt_se->run_list);
  7056. }
  7057. #endif
  7058. void __init sched_init(void)
  7059. {
  7060. int i, j;
  7061. unsigned long alloc_size = 0, ptr;
  7062. #ifdef CONFIG_FAIR_GROUP_SCHED
  7063. alloc_size += 2 * nr_cpu_ids * sizeof(void **);
  7064. #endif
  7065. #ifdef CONFIG_RT_GROUP_SCHED
  7066. alloc_size += 2 * nr_cpu_ids * sizeof(void **);
  7067. #endif
  7068. #ifdef CONFIG_USER_SCHED
  7069. alloc_size *= 2;
  7070. #endif
  7071. /*
  7072. * As sched_init() is called before page_alloc is setup,
  7073. * we use alloc_bootmem().
  7074. */
  7075. if (alloc_size) {
  7076. ptr = (unsigned long)alloc_bootmem(alloc_size);
  7077. #ifdef CONFIG_FAIR_GROUP_SCHED
  7078. init_task_group.se = (struct sched_entity **)ptr;
  7079. ptr += nr_cpu_ids * sizeof(void **);
  7080. init_task_group.cfs_rq = (struct cfs_rq **)ptr;
  7081. ptr += nr_cpu_ids * sizeof(void **);
  7082. #ifdef CONFIG_USER_SCHED
  7083. root_task_group.se = (struct sched_entity **)ptr;
  7084. ptr += nr_cpu_ids * sizeof(void **);
  7085. root_task_group.cfs_rq = (struct cfs_rq **)ptr;
  7086. ptr += nr_cpu_ids * sizeof(void **);
  7087. #endif /* CONFIG_USER_SCHED */
  7088. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7089. #ifdef CONFIG_RT_GROUP_SCHED
  7090. init_task_group.rt_se = (struct sched_rt_entity **)ptr;
  7091. ptr += nr_cpu_ids * sizeof(void **);
  7092. init_task_group.rt_rq = (struct rt_rq **)ptr;
  7093. ptr += nr_cpu_ids * sizeof(void **);
  7094. #ifdef CONFIG_USER_SCHED
  7095. root_task_group.rt_se = (struct sched_rt_entity **)ptr;
  7096. ptr += nr_cpu_ids * sizeof(void **);
  7097. root_task_group.rt_rq = (struct rt_rq **)ptr;
  7098. ptr += nr_cpu_ids * sizeof(void **);
  7099. #endif /* CONFIG_USER_SCHED */
  7100. #endif /* CONFIG_RT_GROUP_SCHED */
  7101. }
  7102. #ifdef CONFIG_SMP
  7103. init_defrootdomain();
  7104. #endif
  7105. init_rt_bandwidth(&def_rt_bandwidth,
  7106. global_rt_period(), global_rt_runtime());
  7107. #ifdef CONFIG_RT_GROUP_SCHED
  7108. init_rt_bandwidth(&init_task_group.rt_bandwidth,
  7109. global_rt_period(), global_rt_runtime());
  7110. #ifdef CONFIG_USER_SCHED
  7111. init_rt_bandwidth(&root_task_group.rt_bandwidth,
  7112. global_rt_period(), RUNTIME_INF);
  7113. #endif /* CONFIG_USER_SCHED */
  7114. #endif /* CONFIG_RT_GROUP_SCHED */
  7115. #ifdef CONFIG_GROUP_SCHED
  7116. list_add(&init_task_group.list, &task_groups);
  7117. INIT_LIST_HEAD(&init_task_group.children);
  7118. #ifdef CONFIG_USER_SCHED
  7119. INIT_LIST_HEAD(&root_task_group.children);
  7120. init_task_group.parent = &root_task_group;
  7121. list_add(&init_task_group.siblings, &root_task_group.children);
  7122. #endif /* CONFIG_USER_SCHED */
  7123. #endif /* CONFIG_GROUP_SCHED */
  7124. for_each_possible_cpu(i) {
  7125. struct rq *rq;
  7126. rq = cpu_rq(i);
  7127. spin_lock_init(&rq->lock);
  7128. rq->nr_running = 0;
  7129. init_cfs_rq(&rq->cfs, rq);
  7130. init_rt_rq(&rq->rt, rq);
  7131. #ifdef CONFIG_FAIR_GROUP_SCHED
  7132. init_task_group.shares = init_task_group_load;
  7133. INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
  7134. #ifdef CONFIG_CGROUP_SCHED
  7135. /*
  7136. * How much cpu bandwidth does init_task_group get?
  7137. *
  7138. * In case of task-groups formed thr' the cgroup filesystem, it
  7139. * gets 100% of the cpu resources in the system. This overall
  7140. * system cpu resource is divided among the tasks of
  7141. * init_task_group and its child task-groups in a fair manner,
  7142. * based on each entity's (task or task-group's) weight
  7143. * (se->load.weight).
  7144. *
  7145. * In other words, if init_task_group has 10 tasks of weight
  7146. * 1024) and two child groups A0 and A1 (of weight 1024 each),
  7147. * then A0's share of the cpu resource is:
  7148. *
  7149. * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
  7150. *
  7151. * We achieve this by letting init_task_group's tasks sit
  7152. * directly in rq->cfs (i.e init_task_group->se[] = NULL).
  7153. */
  7154. init_tg_cfs_entry(&init_task_group, &rq->cfs, NULL, i, 1, NULL);
  7155. #elif defined CONFIG_USER_SCHED
  7156. root_task_group.shares = NICE_0_LOAD;
  7157. init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, 0, NULL);
  7158. /*
  7159. * In case of task-groups formed thr' the user id of tasks,
  7160. * init_task_group represents tasks belonging to root user.
  7161. * Hence it forms a sibling of all subsequent groups formed.
  7162. * In this case, init_task_group gets only a fraction of overall
  7163. * system cpu resource, based on the weight assigned to root
  7164. * user's cpu share (INIT_TASK_GROUP_LOAD). This is accomplished
  7165. * by letting tasks of init_task_group sit in a separate cfs_rq
  7166. * (init_cfs_rq) and having one entity represent this group of
  7167. * tasks in rq->cfs (i.e init_task_group->se[] != NULL).
  7168. */
  7169. init_tg_cfs_entry(&init_task_group,
  7170. &per_cpu(init_cfs_rq, i),
  7171. &per_cpu(init_sched_entity, i), i, 1,
  7172. root_task_group.se[i]);
  7173. #endif
  7174. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7175. rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
  7176. #ifdef CONFIG_RT_GROUP_SCHED
  7177. INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
  7178. #ifdef CONFIG_CGROUP_SCHED
  7179. init_tg_rt_entry(&init_task_group, &rq->rt, NULL, i, 1, NULL);
  7180. #elif defined CONFIG_USER_SCHED
  7181. init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL);
  7182. init_tg_rt_entry(&init_task_group,
  7183. &per_cpu(init_rt_rq, i),
  7184. &per_cpu(init_sched_rt_entity, i), i, 1,
  7185. root_task_group.rt_se[i]);
  7186. #endif
  7187. #endif
  7188. for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
  7189. rq->cpu_load[j] = 0;
  7190. #ifdef CONFIG_SMP
  7191. rq->sd = NULL;
  7192. rq->rd = NULL;
  7193. rq->active_balance = 0;
  7194. rq->next_balance = jiffies;
  7195. rq->push_cpu = 0;
  7196. rq->cpu = i;
  7197. rq->online = 0;
  7198. rq->migration_thread = NULL;
  7199. INIT_LIST_HEAD(&rq->migration_queue);
  7200. rq_attach_root(rq, &def_root_domain);
  7201. #endif
  7202. init_rq_hrtick(rq);
  7203. atomic_set(&rq->nr_iowait, 0);
  7204. }
  7205. set_load_weight(&init_task);
  7206. #ifdef CONFIG_PREEMPT_NOTIFIERS
  7207. INIT_HLIST_HEAD(&init_task.preempt_notifiers);
  7208. #endif
  7209. #ifdef CONFIG_SMP
  7210. open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
  7211. #endif
  7212. #ifdef CONFIG_RT_MUTEXES
  7213. plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
  7214. #endif
  7215. /*
  7216. * The boot idle thread does lazy MMU switching as well:
  7217. */
  7218. atomic_inc(&init_mm.mm_count);
  7219. enter_lazy_tlb(&init_mm, current);
  7220. /*
  7221. * Make us the idle thread. Technically, schedule() should not be
  7222. * called from this thread, however somewhere below it might be,
  7223. * but because we are the idle thread, we just pick up running again
  7224. * when this runqueue becomes "idle".
  7225. */
  7226. init_idle(current, smp_processor_id());
  7227. /*
  7228. * During early bootup we pretend to be a normal task:
  7229. */
  7230. current->sched_class = &fair_sched_class;
  7231. /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
  7232. alloc_bootmem_cpumask_var(&nohz_cpu_mask);
  7233. #ifdef CONFIG_SMP
  7234. #ifdef CONFIG_NO_HZ
  7235. alloc_bootmem_cpumask_var(&nohz.cpu_mask);
  7236. #endif
  7237. alloc_bootmem_cpumask_var(&cpu_isolated_map);
  7238. #endif /* SMP */
  7239. scheduler_running = 1;
  7240. }
  7241. #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
  7242. void __might_sleep(char *file, int line)
  7243. {
  7244. #ifdef in_atomic
  7245. static unsigned long prev_jiffy; /* ratelimiting */
  7246. if ((!in_atomic() && !irqs_disabled()) ||
  7247. system_state != SYSTEM_RUNNING || oops_in_progress)
  7248. return;
  7249. if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
  7250. return;
  7251. prev_jiffy = jiffies;
  7252. printk(KERN_ERR
  7253. "BUG: sleeping function called from invalid context at %s:%d\n",
  7254. file, line);
  7255. printk(KERN_ERR
  7256. "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
  7257. in_atomic(), irqs_disabled(),
  7258. current->pid, current->comm);
  7259. debug_show_held_locks(current);
  7260. if (irqs_disabled())
  7261. print_irqtrace_events(current);
  7262. dump_stack();
  7263. #endif
  7264. }
  7265. EXPORT_SYMBOL(__might_sleep);
  7266. #endif
  7267. #ifdef CONFIG_MAGIC_SYSRQ
  7268. static void normalize_task(struct rq *rq, struct task_struct *p)
  7269. {
  7270. int on_rq;
  7271. update_rq_clock(rq);
  7272. on_rq = p->se.on_rq;
  7273. if (on_rq)
  7274. deactivate_task(rq, p, 0);
  7275. __setscheduler(rq, p, SCHED_NORMAL, 0);
  7276. if (on_rq) {
  7277. activate_task(rq, p, 0);
  7278. resched_task(rq->curr);
  7279. }
  7280. }
  7281. void normalize_rt_tasks(void)
  7282. {
  7283. struct task_struct *g, *p;
  7284. unsigned long flags;
  7285. struct rq *rq;
  7286. read_lock_irqsave(&tasklist_lock, flags);
  7287. do_each_thread(g, p) {
  7288. /*
  7289. * Only normalize user tasks:
  7290. */
  7291. if (!p->mm)
  7292. continue;
  7293. p->se.exec_start = 0;
  7294. #ifdef CONFIG_SCHEDSTATS
  7295. p->se.wait_start = 0;
  7296. p->se.sleep_start = 0;
  7297. p->se.block_start = 0;
  7298. #endif
  7299. if (!rt_task(p)) {
  7300. /*
  7301. * Renice negative nice level userspace
  7302. * tasks back to 0:
  7303. */
  7304. if (TASK_NICE(p) < 0 && p->mm)
  7305. set_user_nice(p, 0);
  7306. continue;
  7307. }
  7308. spin_lock(&p->pi_lock);
  7309. rq = __task_rq_lock(p);
  7310. normalize_task(rq, p);
  7311. __task_rq_unlock(rq);
  7312. spin_unlock(&p->pi_lock);
  7313. } while_each_thread(g, p);
  7314. read_unlock_irqrestore(&tasklist_lock, flags);
  7315. }
  7316. #endif /* CONFIG_MAGIC_SYSRQ */
  7317. #ifdef CONFIG_IA64
  7318. /*
  7319. * These functions are only useful for the IA64 MCA handling.
  7320. *
  7321. * They can only be called when the whole system has been
  7322. * stopped - every CPU needs to be quiescent, and no scheduling
  7323. * activity can take place. Using them for anything else would
  7324. * be a serious bug, and as a result, they aren't even visible
  7325. * under any other configuration.
  7326. */
  7327. /**
  7328. * curr_task - return the current task for a given cpu.
  7329. * @cpu: the processor in question.
  7330. *
  7331. * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
  7332. */
  7333. struct task_struct *curr_task(int cpu)
  7334. {
  7335. return cpu_curr(cpu);
  7336. }
  7337. /**
  7338. * set_curr_task - set the current task for a given cpu.
  7339. * @cpu: the processor in question.
  7340. * @p: the task pointer to set.
  7341. *
  7342. * Description: This function must only be used when non-maskable interrupts
  7343. * are serviced on a separate stack. It allows the architecture to switch the
  7344. * notion of the current task on a cpu in a non-blocking manner. This function
  7345. * must be called with all CPU's synchronized, and interrupts disabled, the
  7346. * and caller must save the original value of the current task (see
  7347. * curr_task() above) and restore that value before reenabling interrupts and
  7348. * re-starting the system.
  7349. *
  7350. * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
  7351. */
  7352. void set_curr_task(int cpu, struct task_struct *p)
  7353. {
  7354. cpu_curr(cpu) = p;
  7355. }
  7356. #endif
  7357. #ifdef CONFIG_FAIR_GROUP_SCHED
  7358. static void free_fair_sched_group(struct task_group *tg)
  7359. {
  7360. int i;
  7361. for_each_possible_cpu(i) {
  7362. if (tg->cfs_rq)
  7363. kfree(tg->cfs_rq[i]);
  7364. if (tg->se)
  7365. kfree(tg->se[i]);
  7366. }
  7367. kfree(tg->cfs_rq);
  7368. kfree(tg->se);
  7369. }
  7370. static
  7371. int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
  7372. {
  7373. struct cfs_rq *cfs_rq;
  7374. struct sched_entity *se;
  7375. struct rq *rq;
  7376. int i;
  7377. tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
  7378. if (!tg->cfs_rq)
  7379. goto err;
  7380. tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
  7381. if (!tg->se)
  7382. goto err;
  7383. tg->shares = NICE_0_LOAD;
  7384. for_each_possible_cpu(i) {
  7385. rq = cpu_rq(i);
  7386. cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
  7387. GFP_KERNEL, cpu_to_node(i));
  7388. if (!cfs_rq)
  7389. goto err;
  7390. se = kzalloc_node(sizeof(struct sched_entity),
  7391. GFP_KERNEL, cpu_to_node(i));
  7392. if (!se)
  7393. goto err;
  7394. init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
  7395. }
  7396. return 1;
  7397. err:
  7398. return 0;
  7399. }
  7400. static inline void register_fair_sched_group(struct task_group *tg, int cpu)
  7401. {
  7402. list_add_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list,
  7403. &cpu_rq(cpu)->leaf_cfs_rq_list);
  7404. }
  7405. static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
  7406. {
  7407. list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
  7408. }
  7409. #else /* !CONFG_FAIR_GROUP_SCHED */
  7410. static inline void free_fair_sched_group(struct task_group *tg)
  7411. {
  7412. }
  7413. static inline
  7414. int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
  7415. {
  7416. return 1;
  7417. }
  7418. static inline void register_fair_sched_group(struct task_group *tg, int cpu)
  7419. {
  7420. }
  7421. static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
  7422. {
  7423. }
  7424. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7425. #ifdef CONFIG_RT_GROUP_SCHED
  7426. static void free_rt_sched_group(struct task_group *tg)
  7427. {
  7428. int i;
  7429. destroy_rt_bandwidth(&tg->rt_bandwidth);
  7430. for_each_possible_cpu(i) {
  7431. if (tg->rt_rq)
  7432. kfree(tg->rt_rq[i]);
  7433. if (tg->rt_se)
  7434. kfree(tg->rt_se[i]);
  7435. }
  7436. kfree(tg->rt_rq);
  7437. kfree(tg->rt_se);
  7438. }
  7439. static
  7440. int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
  7441. {
  7442. struct rt_rq *rt_rq;
  7443. struct sched_rt_entity *rt_se;
  7444. struct rq *rq;
  7445. int i;
  7446. tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
  7447. if (!tg->rt_rq)
  7448. goto err;
  7449. tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
  7450. if (!tg->rt_se)
  7451. goto err;
  7452. init_rt_bandwidth(&tg->rt_bandwidth,
  7453. ktime_to_ns(def_rt_bandwidth.rt_period), 0);
  7454. for_each_possible_cpu(i) {
  7455. rq = cpu_rq(i);
  7456. rt_rq = kzalloc_node(sizeof(struct rt_rq),
  7457. GFP_KERNEL, cpu_to_node(i));
  7458. if (!rt_rq)
  7459. goto err;
  7460. rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
  7461. GFP_KERNEL, cpu_to_node(i));
  7462. if (!rt_se)
  7463. goto err;
  7464. init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
  7465. }
  7466. return 1;
  7467. err:
  7468. return 0;
  7469. }
  7470. static inline void register_rt_sched_group(struct task_group *tg, int cpu)
  7471. {
  7472. list_add_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list,
  7473. &cpu_rq(cpu)->leaf_rt_rq_list);
  7474. }
  7475. static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
  7476. {
  7477. list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
  7478. }
  7479. #else /* !CONFIG_RT_GROUP_SCHED */
  7480. static inline void free_rt_sched_group(struct task_group *tg)
  7481. {
  7482. }
  7483. static inline
  7484. int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
  7485. {
  7486. return 1;
  7487. }
  7488. static inline void register_rt_sched_group(struct task_group *tg, int cpu)
  7489. {
  7490. }
  7491. static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
  7492. {
  7493. }
  7494. #endif /* CONFIG_RT_GROUP_SCHED */
  7495. #ifdef CONFIG_GROUP_SCHED
  7496. static void free_sched_group(struct task_group *tg)
  7497. {
  7498. free_fair_sched_group(tg);
  7499. free_rt_sched_group(tg);
  7500. kfree(tg);
  7501. }
  7502. /* allocate runqueue etc for a new task group */
  7503. struct task_group *sched_create_group(struct task_group *parent)
  7504. {
  7505. struct task_group *tg;
  7506. unsigned long flags;
  7507. int i;
  7508. tg = kzalloc(sizeof(*tg), GFP_KERNEL);
  7509. if (!tg)
  7510. return ERR_PTR(-ENOMEM);
  7511. if (!alloc_fair_sched_group(tg, parent))
  7512. goto err;
  7513. if (!alloc_rt_sched_group(tg, parent))
  7514. goto err;
  7515. spin_lock_irqsave(&task_group_lock, flags);
  7516. for_each_possible_cpu(i) {
  7517. register_fair_sched_group(tg, i);
  7518. register_rt_sched_group(tg, i);
  7519. }
  7520. list_add_rcu(&tg->list, &task_groups);
  7521. WARN_ON(!parent); /* root should already exist */
  7522. tg->parent = parent;
  7523. INIT_LIST_HEAD(&tg->children);
  7524. list_add_rcu(&tg->siblings, &parent->children);
  7525. spin_unlock_irqrestore(&task_group_lock, flags);
  7526. return tg;
  7527. err:
  7528. free_sched_group(tg);
  7529. return ERR_PTR(-ENOMEM);
  7530. }
  7531. /* rcu callback to free various structures associated with a task group */
  7532. static void free_sched_group_rcu(struct rcu_head *rhp)
  7533. {
  7534. /* now it should be safe to free those cfs_rqs */
  7535. free_sched_group(container_of(rhp, struct task_group, rcu));
  7536. }
  7537. /* Destroy runqueue etc associated with a task group */
  7538. void sched_destroy_group(struct task_group *tg)
  7539. {
  7540. unsigned long flags;
  7541. int i;
  7542. spin_lock_irqsave(&task_group_lock, flags);
  7543. for_each_possible_cpu(i) {
  7544. unregister_fair_sched_group(tg, i);
  7545. unregister_rt_sched_group(tg, i);
  7546. }
  7547. list_del_rcu(&tg->list);
  7548. list_del_rcu(&tg->siblings);
  7549. spin_unlock_irqrestore(&task_group_lock, flags);
  7550. /* wait for possible concurrent references to cfs_rqs complete */
  7551. call_rcu(&tg->rcu, free_sched_group_rcu);
  7552. }
  7553. /* change task's runqueue when it moves between groups.
  7554. * The caller of this function should have put the task in its new group
  7555. * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
  7556. * reflect its new group.
  7557. */
  7558. void sched_move_task(struct task_struct *tsk)
  7559. {
  7560. int on_rq, running;
  7561. unsigned long flags;
  7562. struct rq *rq;
  7563. rq = task_rq_lock(tsk, &flags);
  7564. update_rq_clock(rq);
  7565. running = task_current(rq, tsk);
  7566. on_rq = tsk->se.on_rq;
  7567. if (on_rq)
  7568. dequeue_task(rq, tsk, 0);
  7569. if (unlikely(running))
  7570. tsk->sched_class->put_prev_task(rq, tsk);
  7571. set_task_rq(tsk, task_cpu(tsk));
  7572. #ifdef CONFIG_FAIR_GROUP_SCHED
  7573. if (tsk->sched_class->moved_group)
  7574. tsk->sched_class->moved_group(tsk);
  7575. #endif
  7576. if (unlikely(running))
  7577. tsk->sched_class->set_curr_task(rq);
  7578. if (on_rq)
  7579. enqueue_task(rq, tsk, 0);
  7580. task_rq_unlock(rq, &flags);
  7581. }
  7582. #endif /* CONFIG_GROUP_SCHED */
  7583. #ifdef CONFIG_FAIR_GROUP_SCHED
  7584. static void __set_se_shares(struct sched_entity *se, unsigned long shares)
  7585. {
  7586. struct cfs_rq *cfs_rq = se->cfs_rq;
  7587. int on_rq;
  7588. on_rq = se->on_rq;
  7589. if (on_rq)
  7590. dequeue_entity(cfs_rq, se, 0);
  7591. se->load.weight = shares;
  7592. se->load.inv_weight = 0;
  7593. if (on_rq)
  7594. enqueue_entity(cfs_rq, se, 0);
  7595. }
  7596. static void set_se_shares(struct sched_entity *se, unsigned long shares)
  7597. {
  7598. struct cfs_rq *cfs_rq = se->cfs_rq;
  7599. struct rq *rq = cfs_rq->rq;
  7600. unsigned long flags;
  7601. spin_lock_irqsave(&rq->lock, flags);
  7602. __set_se_shares(se, shares);
  7603. spin_unlock_irqrestore(&rq->lock, flags);
  7604. }
  7605. static DEFINE_MUTEX(shares_mutex);
  7606. int sched_group_set_shares(struct task_group *tg, unsigned long shares)
  7607. {
  7608. int i;
  7609. unsigned long flags;
  7610. /*
  7611. * We can't change the weight of the root cgroup.
  7612. */
  7613. if (!tg->se[0])
  7614. return -EINVAL;
  7615. if (shares < MIN_SHARES)
  7616. shares = MIN_SHARES;
  7617. else if (shares > MAX_SHARES)
  7618. shares = MAX_SHARES;
  7619. mutex_lock(&shares_mutex);
  7620. if (tg->shares == shares)
  7621. goto done;
  7622. spin_lock_irqsave(&task_group_lock, flags);
  7623. for_each_possible_cpu(i)
  7624. unregister_fair_sched_group(tg, i);
  7625. list_del_rcu(&tg->siblings);
  7626. spin_unlock_irqrestore(&task_group_lock, flags);
  7627. /* wait for any ongoing reference to this group to finish */
  7628. synchronize_sched();
  7629. /*
  7630. * Now we are free to modify the group's share on each cpu
  7631. * w/o tripping rebalance_share or load_balance_fair.
  7632. */
  7633. tg->shares = shares;
  7634. for_each_possible_cpu(i) {
  7635. /*
  7636. * force a rebalance
  7637. */
  7638. cfs_rq_set_shares(tg->cfs_rq[i], 0);
  7639. set_se_shares(tg->se[i], shares);
  7640. }
  7641. /*
  7642. * Enable load balance activity on this group, by inserting it back on
  7643. * each cpu's rq->leaf_cfs_rq_list.
  7644. */
  7645. spin_lock_irqsave(&task_group_lock, flags);
  7646. for_each_possible_cpu(i)
  7647. register_fair_sched_group(tg, i);
  7648. list_add_rcu(&tg->siblings, &tg->parent->children);
  7649. spin_unlock_irqrestore(&task_group_lock, flags);
  7650. done:
  7651. mutex_unlock(&shares_mutex);
  7652. return 0;
  7653. }
  7654. unsigned long sched_group_shares(struct task_group *tg)
  7655. {
  7656. return tg->shares;
  7657. }
  7658. #endif
  7659. #ifdef CONFIG_RT_GROUP_SCHED
  7660. /*
  7661. * Ensure that the real time constraints are schedulable.
  7662. */
  7663. static DEFINE_MUTEX(rt_constraints_mutex);
  7664. static unsigned long to_ratio(u64 period, u64 runtime)
  7665. {
  7666. if (runtime == RUNTIME_INF)
  7667. return 1ULL << 20;
  7668. return div64_u64(runtime << 20, period);
  7669. }
  7670. /* Must be called with tasklist_lock held */
  7671. static inline int tg_has_rt_tasks(struct task_group *tg)
  7672. {
  7673. struct task_struct *g, *p;
  7674. do_each_thread(g, p) {
  7675. if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg)
  7676. return 1;
  7677. } while_each_thread(g, p);
  7678. return 0;
  7679. }
  7680. struct rt_schedulable_data {
  7681. struct task_group *tg;
  7682. u64 rt_period;
  7683. u64 rt_runtime;
  7684. };
  7685. static int tg_schedulable(struct task_group *tg, void *data)
  7686. {
  7687. struct rt_schedulable_data *d = data;
  7688. struct task_group *child;
  7689. unsigned long total, sum = 0;
  7690. u64 period, runtime;
  7691. period = ktime_to_ns(tg->rt_bandwidth.rt_period);
  7692. runtime = tg->rt_bandwidth.rt_runtime;
  7693. if (tg == d->tg) {
  7694. period = d->rt_period;
  7695. runtime = d->rt_runtime;
  7696. }
  7697. /*
  7698. * Cannot have more runtime than the period.
  7699. */
  7700. if (runtime > period && runtime != RUNTIME_INF)
  7701. return -EINVAL;
  7702. /*
  7703. * Ensure we don't starve existing RT tasks.
  7704. */
  7705. if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
  7706. return -EBUSY;
  7707. total = to_ratio(period, runtime);
  7708. /*
  7709. * Nobody can have more than the global setting allows.
  7710. */
  7711. if (total > to_ratio(global_rt_period(), global_rt_runtime()))
  7712. return -EINVAL;
  7713. /*
  7714. * The sum of our children's runtime should not exceed our own.
  7715. */
  7716. list_for_each_entry_rcu(child, &tg->children, siblings) {
  7717. period = ktime_to_ns(child->rt_bandwidth.rt_period);
  7718. runtime = child->rt_bandwidth.rt_runtime;
  7719. if (child == d->tg) {
  7720. period = d->rt_period;
  7721. runtime = d->rt_runtime;
  7722. }
  7723. sum += to_ratio(period, runtime);
  7724. }
  7725. if (sum > total)
  7726. return -EINVAL;
  7727. return 0;
  7728. }
  7729. static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
  7730. {
  7731. struct rt_schedulable_data data = {
  7732. .tg = tg,
  7733. .rt_period = period,
  7734. .rt_runtime = runtime,
  7735. };
  7736. return walk_tg_tree(tg_schedulable, tg_nop, &data);
  7737. }
  7738. static int tg_set_bandwidth(struct task_group *tg,
  7739. u64 rt_period, u64 rt_runtime)
  7740. {
  7741. int i, err = 0;
  7742. mutex_lock(&rt_constraints_mutex);
  7743. read_lock(&tasklist_lock);
  7744. err = __rt_schedulable(tg, rt_period, rt_runtime);
  7745. if (err)
  7746. goto unlock;
  7747. spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  7748. tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
  7749. tg->rt_bandwidth.rt_runtime = rt_runtime;
  7750. for_each_possible_cpu(i) {
  7751. struct rt_rq *rt_rq = tg->rt_rq[i];
  7752. spin_lock(&rt_rq->rt_runtime_lock);
  7753. rt_rq->rt_runtime = rt_runtime;
  7754. spin_unlock(&rt_rq->rt_runtime_lock);
  7755. }
  7756. spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  7757. unlock:
  7758. read_unlock(&tasklist_lock);
  7759. mutex_unlock(&rt_constraints_mutex);
  7760. return err;
  7761. }
  7762. int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
  7763. {
  7764. u64 rt_runtime, rt_period;
  7765. rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
  7766. rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
  7767. if (rt_runtime_us < 0)
  7768. rt_runtime = RUNTIME_INF;
  7769. return tg_set_bandwidth(tg, rt_period, rt_runtime);
  7770. }
  7771. long sched_group_rt_runtime(struct task_group *tg)
  7772. {
  7773. u64 rt_runtime_us;
  7774. if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
  7775. return -1;
  7776. rt_runtime_us = tg->rt_bandwidth.rt_runtime;
  7777. do_div(rt_runtime_us, NSEC_PER_USEC);
  7778. return rt_runtime_us;
  7779. }
  7780. int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
  7781. {
  7782. u64 rt_runtime, rt_period;
  7783. rt_period = (u64)rt_period_us * NSEC_PER_USEC;
  7784. rt_runtime = tg->rt_bandwidth.rt_runtime;
  7785. if (rt_period == 0)
  7786. return -EINVAL;
  7787. return tg_set_bandwidth(tg, rt_period, rt_runtime);
  7788. }
  7789. long sched_group_rt_period(struct task_group *tg)
  7790. {
  7791. u64 rt_period_us;
  7792. rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
  7793. do_div(rt_period_us, NSEC_PER_USEC);
  7794. return rt_period_us;
  7795. }
  7796. static int sched_rt_global_constraints(void)
  7797. {
  7798. u64 runtime, period;
  7799. int ret = 0;
  7800. if (sysctl_sched_rt_period <= 0)
  7801. return -EINVAL;
  7802. runtime = global_rt_runtime();
  7803. period = global_rt_period();
  7804. /*
  7805. * Sanity check on the sysctl variables.
  7806. */
  7807. if (runtime > period && runtime != RUNTIME_INF)
  7808. return -EINVAL;
  7809. mutex_lock(&rt_constraints_mutex);
  7810. read_lock(&tasklist_lock);
  7811. ret = __rt_schedulable(NULL, 0, 0);
  7812. read_unlock(&tasklist_lock);
  7813. mutex_unlock(&rt_constraints_mutex);
  7814. return ret;
  7815. }
  7816. #else /* !CONFIG_RT_GROUP_SCHED */
  7817. static int sched_rt_global_constraints(void)
  7818. {
  7819. unsigned long flags;
  7820. int i;
  7821. if (sysctl_sched_rt_period <= 0)
  7822. return -EINVAL;
  7823. spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
  7824. for_each_possible_cpu(i) {
  7825. struct rt_rq *rt_rq = &cpu_rq(i)->rt;
  7826. spin_lock(&rt_rq->rt_runtime_lock);
  7827. rt_rq->rt_runtime = global_rt_runtime();
  7828. spin_unlock(&rt_rq->rt_runtime_lock);
  7829. }
  7830. spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
  7831. return 0;
  7832. }
  7833. #endif /* CONFIG_RT_GROUP_SCHED */
  7834. int sched_rt_handler(struct ctl_table *table, int write,
  7835. struct file *filp, void __user *buffer, size_t *lenp,
  7836. loff_t *ppos)
  7837. {
  7838. int ret;
  7839. int old_period, old_runtime;
  7840. static DEFINE_MUTEX(mutex);
  7841. mutex_lock(&mutex);
  7842. old_period = sysctl_sched_rt_period;
  7843. old_runtime = sysctl_sched_rt_runtime;
  7844. ret = proc_dointvec(table, write, filp, buffer, lenp, ppos);
  7845. if (!ret && write) {
  7846. ret = sched_rt_global_constraints();
  7847. if (ret) {
  7848. sysctl_sched_rt_period = old_period;
  7849. sysctl_sched_rt_runtime = old_runtime;
  7850. } else {
  7851. def_rt_bandwidth.rt_runtime = global_rt_runtime();
  7852. def_rt_bandwidth.rt_period =
  7853. ns_to_ktime(global_rt_period());
  7854. }
  7855. }
  7856. mutex_unlock(&mutex);
  7857. return ret;
  7858. }
  7859. #ifdef CONFIG_CGROUP_SCHED
  7860. /* return corresponding task_group object of a cgroup */
  7861. static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
  7862. {
  7863. return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
  7864. struct task_group, css);
  7865. }
  7866. static struct cgroup_subsys_state *
  7867. cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
  7868. {
  7869. struct task_group *tg, *parent;
  7870. if (!cgrp->parent) {
  7871. /* This is early initialization for the top cgroup */
  7872. return &init_task_group.css;
  7873. }
  7874. parent = cgroup_tg(cgrp->parent);
  7875. tg = sched_create_group(parent);
  7876. if (IS_ERR(tg))
  7877. return ERR_PTR(-ENOMEM);
  7878. return &tg->css;
  7879. }
  7880. static void
  7881. cpu_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
  7882. {
  7883. struct task_group *tg = cgroup_tg(cgrp);
  7884. sched_destroy_group(tg);
  7885. }
  7886. static int
  7887. cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
  7888. struct task_struct *tsk)
  7889. {
  7890. #ifdef CONFIG_RT_GROUP_SCHED
  7891. /* Don't accept realtime tasks when there is no way for them to run */
  7892. if (rt_task(tsk) && cgroup_tg(cgrp)->rt_bandwidth.rt_runtime == 0)
  7893. return -EINVAL;
  7894. #else
  7895. /* We don't support RT-tasks being in separate groups */
  7896. if (tsk->sched_class != &fair_sched_class)
  7897. return -EINVAL;
  7898. #endif
  7899. return 0;
  7900. }
  7901. static void
  7902. cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
  7903. struct cgroup *old_cont, struct task_struct *tsk)
  7904. {
  7905. sched_move_task(tsk);
  7906. }
  7907. #ifdef CONFIG_FAIR_GROUP_SCHED
  7908. static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
  7909. u64 shareval)
  7910. {
  7911. return sched_group_set_shares(cgroup_tg(cgrp), shareval);
  7912. }
  7913. static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
  7914. {
  7915. struct task_group *tg = cgroup_tg(cgrp);
  7916. return (u64) tg->shares;
  7917. }
  7918. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7919. #ifdef CONFIG_RT_GROUP_SCHED
  7920. static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
  7921. s64 val)
  7922. {
  7923. return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
  7924. }
  7925. static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
  7926. {
  7927. return sched_group_rt_runtime(cgroup_tg(cgrp));
  7928. }
  7929. static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
  7930. u64 rt_period_us)
  7931. {
  7932. return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
  7933. }
  7934. static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
  7935. {
  7936. return sched_group_rt_period(cgroup_tg(cgrp));
  7937. }
  7938. #endif /* CONFIG_RT_GROUP_SCHED */
  7939. static struct cftype cpu_files[] = {
  7940. #ifdef CONFIG_FAIR_GROUP_SCHED
  7941. {
  7942. .name = "shares",
  7943. .read_u64 = cpu_shares_read_u64,
  7944. .write_u64 = cpu_shares_write_u64,
  7945. },
  7946. #endif
  7947. #ifdef CONFIG_RT_GROUP_SCHED
  7948. {
  7949. .name = "rt_runtime_us",
  7950. .read_s64 = cpu_rt_runtime_read,
  7951. .write_s64 = cpu_rt_runtime_write,
  7952. },
  7953. {
  7954. .name = "rt_period_us",
  7955. .read_u64 = cpu_rt_period_read_uint,
  7956. .write_u64 = cpu_rt_period_write_uint,
  7957. },
  7958. #endif
  7959. };
  7960. static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
  7961. {
  7962. return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files));
  7963. }
  7964. struct cgroup_subsys cpu_cgroup_subsys = {
  7965. .name = "cpu",
  7966. .create = cpu_cgroup_create,
  7967. .destroy = cpu_cgroup_destroy,
  7968. .can_attach = cpu_cgroup_can_attach,
  7969. .attach = cpu_cgroup_attach,
  7970. .populate = cpu_cgroup_populate,
  7971. .subsys_id = cpu_cgroup_subsys_id,
  7972. .early_init = 1,
  7973. };
  7974. #endif /* CONFIG_CGROUP_SCHED */
  7975. #ifdef CONFIG_CGROUP_CPUACCT
  7976. /*
  7977. * CPU accounting code for task groups.
  7978. *
  7979. * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
  7980. * (balbir@in.ibm.com).
  7981. */
  7982. /* track cpu usage of a group of tasks and its child groups */
  7983. struct cpuacct {
  7984. struct cgroup_subsys_state css;
  7985. /* cpuusage holds pointer to a u64-type object on every cpu */
  7986. u64 *cpuusage;
  7987. struct cpuacct *parent;
  7988. };
  7989. struct cgroup_subsys cpuacct_subsys;
  7990. /* return cpu accounting group corresponding to this container */
  7991. static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
  7992. {
  7993. return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
  7994. struct cpuacct, css);
  7995. }
  7996. /* return cpu accounting group to which this task belongs */
  7997. static inline struct cpuacct *task_ca(struct task_struct *tsk)
  7998. {
  7999. return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
  8000. struct cpuacct, css);
  8001. }
  8002. /* create a new cpu accounting group */
  8003. static struct cgroup_subsys_state *cpuacct_create(
  8004. struct cgroup_subsys *ss, struct cgroup *cgrp)
  8005. {
  8006. struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
  8007. if (!ca)
  8008. return ERR_PTR(-ENOMEM);
  8009. ca->cpuusage = alloc_percpu(u64);
  8010. if (!ca->cpuusage) {
  8011. kfree(ca);
  8012. return ERR_PTR(-ENOMEM);
  8013. }
  8014. if (cgrp->parent)
  8015. ca->parent = cgroup_ca(cgrp->parent);
  8016. return &ca->css;
  8017. }
  8018. /* destroy an existing cpu accounting group */
  8019. static void
  8020. cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
  8021. {
  8022. struct cpuacct *ca = cgroup_ca(cgrp);
  8023. free_percpu(ca->cpuusage);
  8024. kfree(ca);
  8025. }
  8026. static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
  8027. {
  8028. u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
  8029. u64 data;
  8030. #ifndef CONFIG_64BIT
  8031. /*
  8032. * Take rq->lock to make 64-bit read safe on 32-bit platforms.
  8033. */
  8034. spin_lock_irq(&cpu_rq(cpu)->lock);
  8035. data = *cpuusage;
  8036. spin_unlock_irq(&cpu_rq(cpu)->lock);
  8037. #else
  8038. data = *cpuusage;
  8039. #endif
  8040. return data;
  8041. }
  8042. static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
  8043. {
  8044. u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
  8045. #ifndef CONFIG_64BIT
  8046. /*
  8047. * Take rq->lock to make 64-bit write safe on 32-bit platforms.
  8048. */
  8049. spin_lock_irq(&cpu_rq(cpu)->lock);
  8050. *cpuusage = val;
  8051. spin_unlock_irq(&cpu_rq(cpu)->lock);
  8052. #else
  8053. *cpuusage = val;
  8054. #endif
  8055. }
  8056. /* return total cpu usage (in nanoseconds) of a group */
  8057. static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
  8058. {
  8059. struct cpuacct *ca = cgroup_ca(cgrp);
  8060. u64 totalcpuusage = 0;
  8061. int i;
  8062. for_each_present_cpu(i)
  8063. totalcpuusage += cpuacct_cpuusage_read(ca, i);
  8064. return totalcpuusage;
  8065. }
  8066. static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
  8067. u64 reset)
  8068. {
  8069. struct cpuacct *ca = cgroup_ca(cgrp);
  8070. int err = 0;
  8071. int i;
  8072. if (reset) {
  8073. err = -EINVAL;
  8074. goto out;
  8075. }
  8076. for_each_present_cpu(i)
  8077. cpuacct_cpuusage_write(ca, i, 0);
  8078. out:
  8079. return err;
  8080. }
  8081. static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
  8082. struct seq_file *m)
  8083. {
  8084. struct cpuacct *ca = cgroup_ca(cgroup);
  8085. u64 percpu;
  8086. int i;
  8087. for_each_present_cpu(i) {
  8088. percpu = cpuacct_cpuusage_read(ca, i);
  8089. seq_printf(m, "%llu ", (unsigned long long) percpu);
  8090. }
  8091. seq_printf(m, "\n");
  8092. return 0;
  8093. }
  8094. static struct cftype files[] = {
  8095. {
  8096. .name = "usage",
  8097. .read_u64 = cpuusage_read,
  8098. .write_u64 = cpuusage_write,
  8099. },
  8100. {
  8101. .name = "usage_percpu",
  8102. .read_seq_string = cpuacct_percpu_seq_read,
  8103. },
  8104. };
  8105. static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
  8106. {
  8107. return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
  8108. }
  8109. /*
  8110. * charge this task's execution time to its accounting group.
  8111. *
  8112. * called with rq->lock held.
  8113. */
  8114. static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
  8115. {
  8116. struct cpuacct *ca;
  8117. int cpu;
  8118. if (!cpuacct_subsys.active)
  8119. return;
  8120. cpu = task_cpu(tsk);
  8121. ca = task_ca(tsk);
  8122. for (; ca; ca = ca->parent) {
  8123. u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
  8124. *cpuusage += cputime;
  8125. }
  8126. }
  8127. struct cgroup_subsys cpuacct_subsys = {
  8128. .name = "cpuacct",
  8129. .create = cpuacct_create,
  8130. .destroy = cpuacct_destroy,
  8131. .populate = cpuacct_populate,
  8132. .subsys_id = cpuacct_subsys_id,
  8133. };
  8134. #endif /* CONFIG_CGROUP_CPUACCT */