workqueue.c 126 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621
  1. /*
  2. * kernel/workqueue.c - generic async execution with shared worker pool
  3. *
  4. * Copyright (C) 2002 Ingo Molnar
  5. *
  6. * Derived from the taskqueue/keventd code by:
  7. * David Woodhouse <dwmw2@infradead.org>
  8. * Andrew Morton
  9. * Kai Petzke <wpp@marie.physik.tu-berlin.de>
  10. * Theodore Ts'o <tytso@mit.edu>
  11. *
  12. * Made to use alloc_percpu by Christoph Lameter.
  13. *
  14. * Copyright (C) 2010 SUSE Linux Products GmbH
  15. * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
  16. *
  17. * This is the generic async execution mechanism. Work items as are
  18. * executed in process context. The worker pool is shared and
  19. * automatically managed. There is one worker pool for each CPU and
  20. * one extra for works which are better served by workers which are
  21. * not bound to any specific CPU.
  22. *
  23. * Please read Documentation/workqueue.txt for details.
  24. */
  25. #include <linux/export.h>
  26. #include <linux/kernel.h>
  27. #include <linux/sched.h>
  28. #include <linux/init.h>
  29. #include <linux/signal.h>
  30. #include <linux/completion.h>
  31. #include <linux/workqueue.h>
  32. #include <linux/slab.h>
  33. #include <linux/cpu.h>
  34. #include <linux/notifier.h>
  35. #include <linux/kthread.h>
  36. #include <linux/hardirq.h>
  37. #include <linux/mempolicy.h>
  38. #include <linux/freezer.h>
  39. #include <linux/kallsyms.h>
  40. #include <linux/debug_locks.h>
  41. #include <linux/lockdep.h>
  42. #include <linux/idr.h>
  43. #include <linux/jhash.h>
  44. #include <linux/hashtable.h>
  45. #include <linux/rculist.h>
  46. #include <linux/nodemask.h>
  47. #include "workqueue_internal.h"
  48. enum {
  49. /*
  50. * worker_pool flags
  51. *
  52. * A bound pool is either associated or disassociated with its CPU.
  53. * While associated (!DISASSOCIATED), all workers are bound to the
  54. * CPU and none has %WORKER_UNBOUND set and concurrency management
  55. * is in effect.
  56. *
  57. * While DISASSOCIATED, the cpu may be offline and all workers have
  58. * %WORKER_UNBOUND set and concurrency management disabled, and may
  59. * be executing on any CPU. The pool behaves as an unbound one.
  60. *
  61. * Note that DISASSOCIATED should be flipped only while holding
  62. * manager_mutex to avoid changing binding state while
  63. * create_worker() is in progress.
  64. */
  65. POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */
  66. POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
  67. POOL_FREEZING = 1 << 3, /* freeze in progress */
  68. /* worker flags */
  69. WORKER_STARTED = 1 << 0, /* started */
  70. WORKER_DIE = 1 << 1, /* die die die */
  71. WORKER_IDLE = 1 << 2, /* is idle */
  72. WORKER_PREP = 1 << 3, /* preparing to run works */
  73. WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
  74. WORKER_UNBOUND = 1 << 7, /* worker is unbound */
  75. WORKER_REBOUND = 1 << 8, /* worker was rebound */
  76. WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE |
  77. WORKER_UNBOUND | WORKER_REBOUND,
  78. NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
  79. UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
  80. BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
  81. MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
  82. IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
  83. MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
  84. /* call for help after 10ms
  85. (min two ticks) */
  86. MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
  87. CREATE_COOLDOWN = HZ, /* time to breath after fail */
  88. /*
  89. * Rescue workers are used only on emergencies and shared by
  90. * all cpus. Give -20.
  91. */
  92. RESCUER_NICE_LEVEL = -20,
  93. HIGHPRI_NICE_LEVEL = -20,
  94. WQ_NAME_LEN = 24,
  95. };
  96. /*
  97. * Structure fields follow one of the following exclusion rules.
  98. *
  99. * I: Modifiable by initialization/destruction paths and read-only for
  100. * everyone else.
  101. *
  102. * P: Preemption protected. Disabling preemption is enough and should
  103. * only be modified and accessed from the local cpu.
  104. *
  105. * L: pool->lock protected. Access with pool->lock held.
  106. *
  107. * X: During normal operation, modification requires pool->lock and should
  108. * be done only from local cpu. Either disabling preemption on local
  109. * cpu or grabbing pool->lock is enough for read access. If
  110. * POOL_DISASSOCIATED is set, it's identical to L.
  111. *
  112. * MG: pool->manager_mutex and pool->lock protected. Writes require both
  113. * locks. Reads can happen under either lock.
  114. *
  115. * PL: wq_pool_mutex protected.
  116. *
  117. * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
  118. *
  119. * WQ: wq->mutex protected.
  120. *
  121. * WR: wq->mutex protected for writes. Sched-RCU protected for reads.
  122. *
  123. * MD: wq_mayday_lock protected.
  124. */
  125. /* struct worker is defined in workqueue_internal.h */
  126. struct worker_pool {
  127. spinlock_t lock; /* the pool lock */
  128. int cpu; /* I: the associated cpu */
  129. int node; /* I: the associated node ID */
  130. int id; /* I: pool ID */
  131. unsigned int flags; /* X: flags */
  132. struct list_head worklist; /* L: list of pending works */
  133. int nr_workers; /* L: total number of workers */
  134. /* nr_idle includes the ones off idle_list for rebinding */
  135. int nr_idle; /* L: currently idle ones */
  136. struct list_head idle_list; /* X: list of idle workers */
  137. struct timer_list idle_timer; /* L: worker idle timeout */
  138. struct timer_list mayday_timer; /* L: SOS timer for workers */
  139. /* a workers is either on busy_hash or idle_list, or the manager */
  140. DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
  141. /* L: hash of busy workers */
  142. /* see manage_workers() for details on the two manager mutexes */
  143. struct mutex manager_arb; /* manager arbitration */
  144. struct mutex manager_mutex; /* manager exclusion */
  145. struct idr worker_idr; /* MG: worker IDs and iteration */
  146. struct workqueue_attrs *attrs; /* I: worker attributes */
  147. struct hlist_node hash_node; /* PL: unbound_pool_hash node */
  148. int refcnt; /* PL: refcnt for unbound pools */
  149. /*
  150. * The current concurrency level. As it's likely to be accessed
  151. * from other CPUs during try_to_wake_up(), put it in a separate
  152. * cacheline.
  153. */
  154. atomic_t nr_running ____cacheline_aligned_in_smp;
  155. /*
  156. * Destruction of pool is sched-RCU protected to allow dereferences
  157. * from get_work_pool().
  158. */
  159. struct rcu_head rcu;
  160. } ____cacheline_aligned_in_smp;
  161. /*
  162. * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
  163. * of work_struct->data are used for flags and the remaining high bits
  164. * point to the pwq; thus, pwqs need to be aligned at two's power of the
  165. * number of flag bits.
  166. */
  167. struct pool_workqueue {
  168. struct worker_pool *pool; /* I: the associated pool */
  169. struct workqueue_struct *wq; /* I: the owning workqueue */
  170. int work_color; /* L: current color */
  171. int flush_color; /* L: flushing color */
  172. int refcnt; /* L: reference count */
  173. int nr_in_flight[WORK_NR_COLORS];
  174. /* L: nr of in_flight works */
  175. int nr_active; /* L: nr of active works */
  176. int max_active; /* L: max active works */
  177. struct list_head delayed_works; /* L: delayed works */
  178. struct list_head pwqs_node; /* WR: node on wq->pwqs */
  179. struct list_head mayday_node; /* MD: node on wq->maydays */
  180. /*
  181. * Release of unbound pwq is punted to system_wq. See put_pwq()
  182. * and pwq_unbound_release_workfn() for details. pool_workqueue
  183. * itself is also sched-RCU protected so that the first pwq can be
  184. * determined without grabbing wq->mutex.
  185. */
  186. struct work_struct unbound_release_work;
  187. struct rcu_head rcu;
  188. } __aligned(1 << WORK_STRUCT_FLAG_BITS);
  189. /*
  190. * Structure used to wait for workqueue flush.
  191. */
  192. struct wq_flusher {
  193. struct list_head list; /* WQ: list of flushers */
  194. int flush_color; /* WQ: flush color waiting for */
  195. struct completion done; /* flush completion */
  196. };
  197. struct wq_device;
  198. /*
  199. * The externally visible workqueue. It relays the issued work items to
  200. * the appropriate worker_pool through its pool_workqueues.
  201. */
  202. struct workqueue_struct {
  203. struct list_head pwqs; /* WR: all pwqs of this wq */
  204. struct list_head list; /* PL: list of all workqueues */
  205. struct mutex mutex; /* protects this wq */
  206. int work_color; /* WQ: current work color */
  207. int flush_color; /* WQ: current flush color */
  208. atomic_t nr_pwqs_to_flush; /* flush in progress */
  209. struct wq_flusher *first_flusher; /* WQ: first flusher */
  210. struct list_head flusher_queue; /* WQ: flush waiters */
  211. struct list_head flusher_overflow; /* WQ: flush overflow list */
  212. struct list_head maydays; /* MD: pwqs requesting rescue */
  213. struct worker *rescuer; /* I: rescue worker */
  214. int nr_drainers; /* WQ: drain in progress */
  215. int saved_max_active; /* WQ: saved pwq max_active */
  216. struct workqueue_attrs *unbound_attrs; /* WQ: only for unbound wqs */
  217. #ifdef CONFIG_SYSFS
  218. struct wq_device *wq_dev; /* I: for sysfs interface */
  219. #endif
  220. #ifdef CONFIG_LOCKDEP
  221. struct lockdep_map lockdep_map;
  222. #endif
  223. char name[WQ_NAME_LEN]; /* I: workqueue name */
  224. /* hot fields used during command issue, aligned to cacheline */
  225. unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
  226. struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
  227. struct pool_workqueue __rcu *numa_pwq_tbl[]; /* FR: unbound pwqs indexed by node */
  228. };
  229. static struct kmem_cache *pwq_cache;
  230. static int wq_numa_tbl_len; /* highest possible NUMA node id + 1 */
  231. static cpumask_var_t *wq_numa_possible_cpumask;
  232. /* possible CPUs of each node */
  233. static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
  234. static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
  235. static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
  236. static LIST_HEAD(workqueues); /* PL: list of all workqueues */
  237. static bool workqueue_freezing; /* PL: have wqs started freezing? */
  238. /* the per-cpu worker pools */
  239. static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
  240. cpu_worker_pools);
  241. static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
  242. /* PL: hash of all unbound pools keyed by pool->attrs */
  243. static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
  244. /* I: attributes used when instantiating standard unbound pools on demand */
  245. static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
  246. struct workqueue_struct *system_wq __read_mostly;
  247. EXPORT_SYMBOL_GPL(system_wq);
  248. struct workqueue_struct *system_highpri_wq __read_mostly;
  249. EXPORT_SYMBOL_GPL(system_highpri_wq);
  250. struct workqueue_struct *system_long_wq __read_mostly;
  251. EXPORT_SYMBOL_GPL(system_long_wq);
  252. struct workqueue_struct *system_unbound_wq __read_mostly;
  253. EXPORT_SYMBOL_GPL(system_unbound_wq);
  254. struct workqueue_struct *system_freezable_wq __read_mostly;
  255. EXPORT_SYMBOL_GPL(system_freezable_wq);
  256. static int worker_thread(void *__worker);
  257. static void copy_workqueue_attrs(struct workqueue_attrs *to,
  258. const struct workqueue_attrs *from);
  259. #define CREATE_TRACE_POINTS
  260. #include <trace/events/workqueue.h>
  261. #define assert_rcu_or_pool_mutex() \
  262. rcu_lockdep_assert(rcu_read_lock_sched_held() || \
  263. lockdep_is_held(&wq_pool_mutex), \
  264. "sched RCU or wq_pool_mutex should be held")
  265. #define assert_rcu_or_wq_mutex(wq) \
  266. rcu_lockdep_assert(rcu_read_lock_sched_held() || \
  267. lockdep_is_held(&wq->mutex), \
  268. "sched RCU or wq->mutex should be held")
  269. #ifdef CONFIG_LOCKDEP
  270. #define assert_manager_or_pool_lock(pool) \
  271. WARN_ONCE(debug_locks && \
  272. !lockdep_is_held(&(pool)->manager_mutex) && \
  273. !lockdep_is_held(&(pool)->lock), \
  274. "pool->manager_mutex or ->lock should be held")
  275. #else
  276. #define assert_manager_or_pool_lock(pool) do { } while (0)
  277. #endif
  278. #define for_each_cpu_worker_pool(pool, cpu) \
  279. for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
  280. (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
  281. (pool)++)
  282. /**
  283. * for_each_pool - iterate through all worker_pools in the system
  284. * @pool: iteration cursor
  285. * @pi: integer used for iteration
  286. *
  287. * This must be called either with wq_pool_mutex held or sched RCU read
  288. * locked. If the pool needs to be used beyond the locking in effect, the
  289. * caller is responsible for guaranteeing that the pool stays online.
  290. *
  291. * The if/else clause exists only for the lockdep assertion and can be
  292. * ignored.
  293. */
  294. #define for_each_pool(pool, pi) \
  295. idr_for_each_entry(&worker_pool_idr, pool, pi) \
  296. if (({ assert_rcu_or_pool_mutex(); false; })) { } \
  297. else
  298. /**
  299. * for_each_pool_worker - iterate through all workers of a worker_pool
  300. * @worker: iteration cursor
  301. * @wi: integer used for iteration
  302. * @pool: worker_pool to iterate workers of
  303. *
  304. * This must be called with either @pool->manager_mutex or ->lock held.
  305. *
  306. * The if/else clause exists only for the lockdep assertion and can be
  307. * ignored.
  308. */
  309. #define for_each_pool_worker(worker, wi, pool) \
  310. idr_for_each_entry(&(pool)->worker_idr, (worker), (wi)) \
  311. if (({ assert_manager_or_pool_lock((pool)); false; })) { } \
  312. else
  313. /**
  314. * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
  315. * @pwq: iteration cursor
  316. * @wq: the target workqueue
  317. *
  318. * This must be called either with wq->mutex held or sched RCU read locked.
  319. * If the pwq needs to be used beyond the locking in effect, the caller is
  320. * responsible for guaranteeing that the pwq stays online.
  321. *
  322. * The if/else clause exists only for the lockdep assertion and can be
  323. * ignored.
  324. */
  325. #define for_each_pwq(pwq, wq) \
  326. list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
  327. if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
  328. else
  329. #ifdef CONFIG_DEBUG_OBJECTS_WORK
  330. static struct debug_obj_descr work_debug_descr;
  331. static void *work_debug_hint(void *addr)
  332. {
  333. return ((struct work_struct *) addr)->func;
  334. }
  335. /*
  336. * fixup_init is called when:
  337. * - an active object is initialized
  338. */
  339. static int work_fixup_init(void *addr, enum debug_obj_state state)
  340. {
  341. struct work_struct *work = addr;
  342. switch (state) {
  343. case ODEBUG_STATE_ACTIVE:
  344. cancel_work_sync(work);
  345. debug_object_init(work, &work_debug_descr);
  346. return 1;
  347. default:
  348. return 0;
  349. }
  350. }
  351. /*
  352. * fixup_activate is called when:
  353. * - an active object is activated
  354. * - an unknown object is activated (might be a statically initialized object)
  355. */
  356. static int work_fixup_activate(void *addr, enum debug_obj_state state)
  357. {
  358. struct work_struct *work = addr;
  359. switch (state) {
  360. case ODEBUG_STATE_NOTAVAILABLE:
  361. /*
  362. * This is not really a fixup. The work struct was
  363. * statically initialized. We just make sure that it
  364. * is tracked in the object tracker.
  365. */
  366. if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
  367. debug_object_init(work, &work_debug_descr);
  368. debug_object_activate(work, &work_debug_descr);
  369. return 0;
  370. }
  371. WARN_ON_ONCE(1);
  372. return 0;
  373. case ODEBUG_STATE_ACTIVE:
  374. WARN_ON(1);
  375. default:
  376. return 0;
  377. }
  378. }
  379. /*
  380. * fixup_free is called when:
  381. * - an active object is freed
  382. */
  383. static int work_fixup_free(void *addr, enum debug_obj_state state)
  384. {
  385. struct work_struct *work = addr;
  386. switch (state) {
  387. case ODEBUG_STATE_ACTIVE:
  388. cancel_work_sync(work);
  389. debug_object_free(work, &work_debug_descr);
  390. return 1;
  391. default:
  392. return 0;
  393. }
  394. }
  395. static struct debug_obj_descr work_debug_descr = {
  396. .name = "work_struct",
  397. .debug_hint = work_debug_hint,
  398. .fixup_init = work_fixup_init,
  399. .fixup_activate = work_fixup_activate,
  400. .fixup_free = work_fixup_free,
  401. };
  402. static inline void debug_work_activate(struct work_struct *work)
  403. {
  404. debug_object_activate(work, &work_debug_descr);
  405. }
  406. static inline void debug_work_deactivate(struct work_struct *work)
  407. {
  408. debug_object_deactivate(work, &work_debug_descr);
  409. }
  410. void __init_work(struct work_struct *work, int onstack)
  411. {
  412. if (onstack)
  413. debug_object_init_on_stack(work, &work_debug_descr);
  414. else
  415. debug_object_init(work, &work_debug_descr);
  416. }
  417. EXPORT_SYMBOL_GPL(__init_work);
  418. void destroy_work_on_stack(struct work_struct *work)
  419. {
  420. debug_object_free(work, &work_debug_descr);
  421. }
  422. EXPORT_SYMBOL_GPL(destroy_work_on_stack);
  423. #else
  424. static inline void debug_work_activate(struct work_struct *work) { }
  425. static inline void debug_work_deactivate(struct work_struct *work) { }
  426. #endif
  427. /* allocate ID and assign it to @pool */
  428. static int worker_pool_assign_id(struct worker_pool *pool)
  429. {
  430. int ret;
  431. lockdep_assert_held(&wq_pool_mutex);
  432. do {
  433. if (!idr_pre_get(&worker_pool_idr, GFP_KERNEL))
  434. return -ENOMEM;
  435. ret = idr_get_new(&worker_pool_idr, pool, &pool->id);
  436. } while (ret == -EAGAIN);
  437. return ret;
  438. }
  439. /**
  440. * first_pwq - return the first pool_workqueue of the specified workqueue
  441. * @wq: the target workqueue
  442. *
  443. * This must be called either with wq->mutex held or sched RCU read locked.
  444. * If the pwq needs to be used beyond the locking in effect, the caller is
  445. * responsible for guaranteeing that the pwq stays online.
  446. */
  447. static struct pool_workqueue *first_pwq(struct workqueue_struct *wq)
  448. {
  449. assert_rcu_or_wq_mutex(wq);
  450. return list_first_or_null_rcu(&wq->pwqs, struct pool_workqueue,
  451. pwqs_node);
  452. }
  453. /**
  454. * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
  455. * @wq: the target workqueue
  456. * @node: the node ID
  457. *
  458. * This must be called either with pwq_lock held or sched RCU read locked.
  459. * If the pwq needs to be used beyond the locking in effect, the caller is
  460. * responsible for guaranteeing that the pwq stays online.
  461. */
  462. static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
  463. int node)
  464. {
  465. assert_rcu_or_wq_mutex(wq);
  466. return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
  467. }
  468. static unsigned int work_color_to_flags(int color)
  469. {
  470. return color << WORK_STRUCT_COLOR_SHIFT;
  471. }
  472. static int get_work_color(struct work_struct *work)
  473. {
  474. return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
  475. ((1 << WORK_STRUCT_COLOR_BITS) - 1);
  476. }
  477. static int work_next_color(int color)
  478. {
  479. return (color + 1) % WORK_NR_COLORS;
  480. }
  481. /*
  482. * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
  483. * contain the pointer to the queued pwq. Once execution starts, the flag
  484. * is cleared and the high bits contain OFFQ flags and pool ID.
  485. *
  486. * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
  487. * and clear_work_data() can be used to set the pwq, pool or clear
  488. * work->data. These functions should only be called while the work is
  489. * owned - ie. while the PENDING bit is set.
  490. *
  491. * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
  492. * corresponding to a work. Pool is available once the work has been
  493. * queued anywhere after initialization until it is sync canceled. pwq is
  494. * available only while the work item is queued.
  495. *
  496. * %WORK_OFFQ_CANCELING is used to mark a work item which is being
  497. * canceled. While being canceled, a work item may have its PENDING set
  498. * but stay off timer and worklist for arbitrarily long and nobody should
  499. * try to steal the PENDING bit.
  500. */
  501. static inline void set_work_data(struct work_struct *work, unsigned long data,
  502. unsigned long flags)
  503. {
  504. WARN_ON_ONCE(!work_pending(work));
  505. atomic_long_set(&work->data, data | flags | work_static(work));
  506. }
  507. static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
  508. unsigned long extra_flags)
  509. {
  510. set_work_data(work, (unsigned long)pwq,
  511. WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
  512. }
  513. static void set_work_pool_and_keep_pending(struct work_struct *work,
  514. int pool_id)
  515. {
  516. set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
  517. WORK_STRUCT_PENDING);
  518. }
  519. static void set_work_pool_and_clear_pending(struct work_struct *work,
  520. int pool_id)
  521. {
  522. /*
  523. * The following wmb is paired with the implied mb in
  524. * test_and_set_bit(PENDING) and ensures all updates to @work made
  525. * here are visible to and precede any updates by the next PENDING
  526. * owner.
  527. */
  528. smp_wmb();
  529. set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
  530. }
  531. static void clear_work_data(struct work_struct *work)
  532. {
  533. smp_wmb(); /* see set_work_pool_and_clear_pending() */
  534. set_work_data(work, WORK_STRUCT_NO_POOL, 0);
  535. }
  536. static struct pool_workqueue *get_work_pwq(struct work_struct *work)
  537. {
  538. unsigned long data = atomic_long_read(&work->data);
  539. if (data & WORK_STRUCT_PWQ)
  540. return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
  541. else
  542. return NULL;
  543. }
  544. /**
  545. * get_work_pool - return the worker_pool a given work was associated with
  546. * @work: the work item of interest
  547. *
  548. * Return the worker_pool @work was last associated with. %NULL if none.
  549. *
  550. * Pools are created and destroyed under wq_pool_mutex, and allows read
  551. * access under sched-RCU read lock. As such, this function should be
  552. * called under wq_pool_mutex or with preemption disabled.
  553. *
  554. * All fields of the returned pool are accessible as long as the above
  555. * mentioned locking is in effect. If the returned pool needs to be used
  556. * beyond the critical section, the caller is responsible for ensuring the
  557. * returned pool is and stays online.
  558. */
  559. static struct worker_pool *get_work_pool(struct work_struct *work)
  560. {
  561. unsigned long data = atomic_long_read(&work->data);
  562. int pool_id;
  563. assert_rcu_or_pool_mutex();
  564. if (data & WORK_STRUCT_PWQ)
  565. return ((struct pool_workqueue *)
  566. (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
  567. pool_id = data >> WORK_OFFQ_POOL_SHIFT;
  568. if (pool_id == WORK_OFFQ_POOL_NONE)
  569. return NULL;
  570. return idr_find(&worker_pool_idr, pool_id);
  571. }
  572. /**
  573. * get_work_pool_id - return the worker pool ID a given work is associated with
  574. * @work: the work item of interest
  575. *
  576. * Return the worker_pool ID @work was last associated with.
  577. * %WORK_OFFQ_POOL_NONE if none.
  578. */
  579. static int get_work_pool_id(struct work_struct *work)
  580. {
  581. unsigned long data = atomic_long_read(&work->data);
  582. if (data & WORK_STRUCT_PWQ)
  583. return ((struct pool_workqueue *)
  584. (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
  585. return data >> WORK_OFFQ_POOL_SHIFT;
  586. }
  587. static void mark_work_canceling(struct work_struct *work)
  588. {
  589. unsigned long pool_id = get_work_pool_id(work);
  590. pool_id <<= WORK_OFFQ_POOL_SHIFT;
  591. set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
  592. }
  593. static bool work_is_canceling(struct work_struct *work)
  594. {
  595. unsigned long data = atomic_long_read(&work->data);
  596. return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
  597. }
  598. /*
  599. * Policy functions. These define the policies on how the global worker
  600. * pools are managed. Unless noted otherwise, these functions assume that
  601. * they're being called with pool->lock held.
  602. */
  603. static bool __need_more_worker(struct worker_pool *pool)
  604. {
  605. return !atomic_read(&pool->nr_running);
  606. }
  607. /*
  608. * Need to wake up a worker? Called from anything but currently
  609. * running workers.
  610. *
  611. * Note that, because unbound workers never contribute to nr_running, this
  612. * function will always return %true for unbound pools as long as the
  613. * worklist isn't empty.
  614. */
  615. static bool need_more_worker(struct worker_pool *pool)
  616. {
  617. return !list_empty(&pool->worklist) && __need_more_worker(pool);
  618. }
  619. /* Can I start working? Called from busy but !running workers. */
  620. static bool may_start_working(struct worker_pool *pool)
  621. {
  622. return pool->nr_idle;
  623. }
  624. /* Do I need to keep working? Called from currently running workers. */
  625. static bool keep_working(struct worker_pool *pool)
  626. {
  627. return !list_empty(&pool->worklist) &&
  628. atomic_read(&pool->nr_running) <= 1;
  629. }
  630. /* Do we need a new worker? Called from manager. */
  631. static bool need_to_create_worker(struct worker_pool *pool)
  632. {
  633. return need_more_worker(pool) && !may_start_working(pool);
  634. }
  635. /* Do I need to be the manager? */
  636. static bool need_to_manage_workers(struct worker_pool *pool)
  637. {
  638. return need_to_create_worker(pool) ||
  639. (pool->flags & POOL_MANAGE_WORKERS);
  640. }
  641. /* Do we have too many workers and should some go away? */
  642. static bool too_many_workers(struct worker_pool *pool)
  643. {
  644. bool managing = mutex_is_locked(&pool->manager_arb);
  645. int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
  646. int nr_busy = pool->nr_workers - nr_idle;
  647. /*
  648. * nr_idle and idle_list may disagree if idle rebinding is in
  649. * progress. Never return %true if idle_list is empty.
  650. */
  651. if (list_empty(&pool->idle_list))
  652. return false;
  653. return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
  654. }
  655. /*
  656. * Wake up functions.
  657. */
  658. /* Return the first worker. Safe with preemption disabled */
  659. static struct worker *first_worker(struct worker_pool *pool)
  660. {
  661. if (unlikely(list_empty(&pool->idle_list)))
  662. return NULL;
  663. return list_first_entry(&pool->idle_list, struct worker, entry);
  664. }
  665. /**
  666. * wake_up_worker - wake up an idle worker
  667. * @pool: worker pool to wake worker from
  668. *
  669. * Wake up the first idle worker of @pool.
  670. *
  671. * CONTEXT:
  672. * spin_lock_irq(pool->lock).
  673. */
  674. static void wake_up_worker(struct worker_pool *pool)
  675. {
  676. struct worker *worker = first_worker(pool);
  677. if (likely(worker))
  678. wake_up_process(worker->task);
  679. }
  680. /**
  681. * wq_worker_waking_up - a worker is waking up
  682. * @task: task waking up
  683. * @cpu: CPU @task is waking up to
  684. *
  685. * This function is called during try_to_wake_up() when a worker is
  686. * being awoken.
  687. *
  688. * CONTEXT:
  689. * spin_lock_irq(rq->lock)
  690. */
  691. void wq_worker_waking_up(struct task_struct *task, int cpu)
  692. {
  693. struct worker *worker = kthread_data(task);
  694. if (!(worker->flags & WORKER_NOT_RUNNING)) {
  695. WARN_ON_ONCE(worker->pool->cpu != cpu);
  696. atomic_inc(&worker->pool->nr_running);
  697. }
  698. }
  699. /**
  700. * wq_worker_sleeping - a worker is going to sleep
  701. * @task: task going to sleep
  702. * @cpu: CPU in question, must be the current CPU number
  703. *
  704. * This function is called during schedule() when a busy worker is
  705. * going to sleep. Worker on the same cpu can be woken up by
  706. * returning pointer to its task.
  707. *
  708. * CONTEXT:
  709. * spin_lock_irq(rq->lock)
  710. *
  711. * RETURNS:
  712. * Worker task on @cpu to wake up, %NULL if none.
  713. */
  714. struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu)
  715. {
  716. struct worker *worker = kthread_data(task), *to_wakeup = NULL;
  717. struct worker_pool *pool;
  718. /*
  719. * Rescuers, which may not have all the fields set up like normal
  720. * workers, also reach here, let's not access anything before
  721. * checking NOT_RUNNING.
  722. */
  723. if (worker->flags & WORKER_NOT_RUNNING)
  724. return NULL;
  725. pool = worker->pool;
  726. /* this can only happen on the local cpu */
  727. if (WARN_ON_ONCE(cpu != raw_smp_processor_id()))
  728. return NULL;
  729. /*
  730. * The counterpart of the following dec_and_test, implied mb,
  731. * worklist not empty test sequence is in insert_work().
  732. * Please read comment there.
  733. *
  734. * NOT_RUNNING is clear. This means that we're bound to and
  735. * running on the local cpu w/ rq lock held and preemption
  736. * disabled, which in turn means that none else could be
  737. * manipulating idle_list, so dereferencing idle_list without pool
  738. * lock is safe.
  739. */
  740. if (atomic_dec_and_test(&pool->nr_running) &&
  741. !list_empty(&pool->worklist))
  742. to_wakeup = first_worker(pool);
  743. return to_wakeup ? to_wakeup->task : NULL;
  744. }
  745. /**
  746. * worker_set_flags - set worker flags and adjust nr_running accordingly
  747. * @worker: self
  748. * @flags: flags to set
  749. * @wakeup: wakeup an idle worker if necessary
  750. *
  751. * Set @flags in @worker->flags and adjust nr_running accordingly. If
  752. * nr_running becomes zero and @wakeup is %true, an idle worker is
  753. * woken up.
  754. *
  755. * CONTEXT:
  756. * spin_lock_irq(pool->lock)
  757. */
  758. static inline void worker_set_flags(struct worker *worker, unsigned int flags,
  759. bool wakeup)
  760. {
  761. struct worker_pool *pool = worker->pool;
  762. WARN_ON_ONCE(worker->task != current);
  763. /*
  764. * If transitioning into NOT_RUNNING, adjust nr_running and
  765. * wake up an idle worker as necessary if requested by
  766. * @wakeup.
  767. */
  768. if ((flags & WORKER_NOT_RUNNING) &&
  769. !(worker->flags & WORKER_NOT_RUNNING)) {
  770. if (wakeup) {
  771. if (atomic_dec_and_test(&pool->nr_running) &&
  772. !list_empty(&pool->worklist))
  773. wake_up_worker(pool);
  774. } else
  775. atomic_dec(&pool->nr_running);
  776. }
  777. worker->flags |= flags;
  778. }
  779. /**
  780. * worker_clr_flags - clear worker flags and adjust nr_running accordingly
  781. * @worker: self
  782. * @flags: flags to clear
  783. *
  784. * Clear @flags in @worker->flags and adjust nr_running accordingly.
  785. *
  786. * CONTEXT:
  787. * spin_lock_irq(pool->lock)
  788. */
  789. static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
  790. {
  791. struct worker_pool *pool = worker->pool;
  792. unsigned int oflags = worker->flags;
  793. WARN_ON_ONCE(worker->task != current);
  794. worker->flags &= ~flags;
  795. /*
  796. * If transitioning out of NOT_RUNNING, increment nr_running. Note
  797. * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
  798. * of multiple flags, not a single flag.
  799. */
  800. if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
  801. if (!(worker->flags & WORKER_NOT_RUNNING))
  802. atomic_inc(&pool->nr_running);
  803. }
  804. /**
  805. * find_worker_executing_work - find worker which is executing a work
  806. * @pool: pool of interest
  807. * @work: work to find worker for
  808. *
  809. * Find a worker which is executing @work on @pool by searching
  810. * @pool->busy_hash which is keyed by the address of @work. For a worker
  811. * to match, its current execution should match the address of @work and
  812. * its work function. This is to avoid unwanted dependency between
  813. * unrelated work executions through a work item being recycled while still
  814. * being executed.
  815. *
  816. * This is a bit tricky. A work item may be freed once its execution
  817. * starts and nothing prevents the freed area from being recycled for
  818. * another work item. If the same work item address ends up being reused
  819. * before the original execution finishes, workqueue will identify the
  820. * recycled work item as currently executing and make it wait until the
  821. * current execution finishes, introducing an unwanted dependency.
  822. *
  823. * This function checks the work item address and work function to avoid
  824. * false positives. Note that this isn't complete as one may construct a
  825. * work function which can introduce dependency onto itself through a
  826. * recycled work item. Well, if somebody wants to shoot oneself in the
  827. * foot that badly, there's only so much we can do, and if such deadlock
  828. * actually occurs, it should be easy to locate the culprit work function.
  829. *
  830. * CONTEXT:
  831. * spin_lock_irq(pool->lock).
  832. *
  833. * RETURNS:
  834. * Pointer to worker which is executing @work if found, NULL
  835. * otherwise.
  836. */
  837. static struct worker *find_worker_executing_work(struct worker_pool *pool,
  838. struct work_struct *work)
  839. {
  840. struct worker *worker;
  841. hash_for_each_possible(pool->busy_hash, worker, hentry,
  842. (unsigned long)work)
  843. if (worker->current_work == work &&
  844. worker->current_func == work->func)
  845. return worker;
  846. return NULL;
  847. }
  848. /**
  849. * move_linked_works - move linked works to a list
  850. * @work: start of series of works to be scheduled
  851. * @head: target list to append @work to
  852. * @nextp: out paramter for nested worklist walking
  853. *
  854. * Schedule linked works starting from @work to @head. Work series to
  855. * be scheduled starts at @work and includes any consecutive work with
  856. * WORK_STRUCT_LINKED set in its predecessor.
  857. *
  858. * If @nextp is not NULL, it's updated to point to the next work of
  859. * the last scheduled work. This allows move_linked_works() to be
  860. * nested inside outer list_for_each_entry_safe().
  861. *
  862. * CONTEXT:
  863. * spin_lock_irq(pool->lock).
  864. */
  865. static void move_linked_works(struct work_struct *work, struct list_head *head,
  866. struct work_struct **nextp)
  867. {
  868. struct work_struct *n;
  869. /*
  870. * Linked worklist will always end before the end of the list,
  871. * use NULL for list head.
  872. */
  873. list_for_each_entry_safe_from(work, n, NULL, entry) {
  874. list_move_tail(&work->entry, head);
  875. if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
  876. break;
  877. }
  878. /*
  879. * If we're already inside safe list traversal and have moved
  880. * multiple works to the scheduled queue, the next position
  881. * needs to be updated.
  882. */
  883. if (nextp)
  884. *nextp = n;
  885. }
  886. /**
  887. * get_pwq - get an extra reference on the specified pool_workqueue
  888. * @pwq: pool_workqueue to get
  889. *
  890. * Obtain an extra reference on @pwq. The caller should guarantee that
  891. * @pwq has positive refcnt and be holding the matching pool->lock.
  892. */
  893. static void get_pwq(struct pool_workqueue *pwq)
  894. {
  895. lockdep_assert_held(&pwq->pool->lock);
  896. WARN_ON_ONCE(pwq->refcnt <= 0);
  897. pwq->refcnt++;
  898. }
  899. /**
  900. * put_pwq - put a pool_workqueue reference
  901. * @pwq: pool_workqueue to put
  902. *
  903. * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
  904. * destruction. The caller should be holding the matching pool->lock.
  905. */
  906. static void put_pwq(struct pool_workqueue *pwq)
  907. {
  908. lockdep_assert_held(&pwq->pool->lock);
  909. if (likely(--pwq->refcnt))
  910. return;
  911. if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
  912. return;
  913. /*
  914. * @pwq can't be released under pool->lock, bounce to
  915. * pwq_unbound_release_workfn(). This never recurses on the same
  916. * pool->lock as this path is taken only for unbound workqueues and
  917. * the release work item is scheduled on a per-cpu workqueue. To
  918. * avoid lockdep warning, unbound pool->locks are given lockdep
  919. * subclass of 1 in get_unbound_pool().
  920. */
  921. schedule_work(&pwq->unbound_release_work);
  922. }
  923. /**
  924. * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
  925. * @pwq: pool_workqueue to put (can be %NULL)
  926. *
  927. * put_pwq() with locking. This function also allows %NULL @pwq.
  928. */
  929. static void put_pwq_unlocked(struct pool_workqueue *pwq)
  930. {
  931. if (pwq) {
  932. /*
  933. * As both pwqs and pools are sched-RCU protected, the
  934. * following lock operations are safe.
  935. */
  936. spin_lock_irq(&pwq->pool->lock);
  937. put_pwq(pwq);
  938. spin_unlock_irq(&pwq->pool->lock);
  939. }
  940. }
  941. static void pwq_activate_delayed_work(struct work_struct *work)
  942. {
  943. struct pool_workqueue *pwq = get_work_pwq(work);
  944. trace_workqueue_activate_work(work);
  945. move_linked_works(work, &pwq->pool->worklist, NULL);
  946. __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
  947. pwq->nr_active++;
  948. }
  949. static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
  950. {
  951. struct work_struct *work = list_first_entry(&pwq->delayed_works,
  952. struct work_struct, entry);
  953. pwq_activate_delayed_work(work);
  954. }
  955. /**
  956. * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
  957. * @pwq: pwq of interest
  958. * @color: color of work which left the queue
  959. *
  960. * A work either has completed or is removed from pending queue,
  961. * decrement nr_in_flight of its pwq and handle workqueue flushing.
  962. *
  963. * CONTEXT:
  964. * spin_lock_irq(pool->lock).
  965. */
  966. static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
  967. {
  968. /* uncolored work items don't participate in flushing or nr_active */
  969. if (color == WORK_NO_COLOR)
  970. goto out_put;
  971. pwq->nr_in_flight[color]--;
  972. pwq->nr_active--;
  973. if (!list_empty(&pwq->delayed_works)) {
  974. /* one down, submit a delayed one */
  975. if (pwq->nr_active < pwq->max_active)
  976. pwq_activate_first_delayed(pwq);
  977. }
  978. /* is flush in progress and are we at the flushing tip? */
  979. if (likely(pwq->flush_color != color))
  980. goto out_put;
  981. /* are there still in-flight works? */
  982. if (pwq->nr_in_flight[color])
  983. goto out_put;
  984. /* this pwq is done, clear flush_color */
  985. pwq->flush_color = -1;
  986. /*
  987. * If this was the last pwq, wake up the first flusher. It
  988. * will handle the rest.
  989. */
  990. if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
  991. complete(&pwq->wq->first_flusher->done);
  992. out_put:
  993. put_pwq(pwq);
  994. }
  995. /**
  996. * try_to_grab_pending - steal work item from worklist and disable irq
  997. * @work: work item to steal
  998. * @is_dwork: @work is a delayed_work
  999. * @flags: place to store irq state
  1000. *
  1001. * Try to grab PENDING bit of @work. This function can handle @work in any
  1002. * stable state - idle, on timer or on worklist. Return values are
  1003. *
  1004. * 1 if @work was pending and we successfully stole PENDING
  1005. * 0 if @work was idle and we claimed PENDING
  1006. * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
  1007. * -ENOENT if someone else is canceling @work, this state may persist
  1008. * for arbitrarily long
  1009. *
  1010. * On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
  1011. * interrupted while holding PENDING and @work off queue, irq must be
  1012. * disabled on entry. This, combined with delayed_work->timer being
  1013. * irqsafe, ensures that we return -EAGAIN for finite short period of time.
  1014. *
  1015. * On successful return, >= 0, irq is disabled and the caller is
  1016. * responsible for releasing it using local_irq_restore(*@flags).
  1017. *
  1018. * This function is safe to call from any context including IRQ handler.
  1019. */
  1020. static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
  1021. unsigned long *flags)
  1022. {
  1023. struct worker_pool *pool;
  1024. struct pool_workqueue *pwq;
  1025. local_irq_save(*flags);
  1026. /* try to steal the timer if it exists */
  1027. if (is_dwork) {
  1028. struct delayed_work *dwork = to_delayed_work(work);
  1029. /*
  1030. * dwork->timer is irqsafe. If del_timer() fails, it's
  1031. * guaranteed that the timer is not queued anywhere and not
  1032. * running on the local CPU.
  1033. */
  1034. if (likely(del_timer(&dwork->timer)))
  1035. return 1;
  1036. }
  1037. /* try to claim PENDING the normal way */
  1038. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
  1039. return 0;
  1040. /*
  1041. * The queueing is in progress, or it is already queued. Try to
  1042. * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
  1043. */
  1044. pool = get_work_pool(work);
  1045. if (!pool)
  1046. goto fail;
  1047. spin_lock(&pool->lock);
  1048. /*
  1049. * work->data is guaranteed to point to pwq only while the work
  1050. * item is queued on pwq->wq, and both updating work->data to point
  1051. * to pwq on queueing and to pool on dequeueing are done under
  1052. * pwq->pool->lock. This in turn guarantees that, if work->data
  1053. * points to pwq which is associated with a locked pool, the work
  1054. * item is currently queued on that pool.
  1055. */
  1056. pwq = get_work_pwq(work);
  1057. if (pwq && pwq->pool == pool) {
  1058. debug_work_deactivate(work);
  1059. /*
  1060. * A delayed work item cannot be grabbed directly because
  1061. * it might have linked NO_COLOR work items which, if left
  1062. * on the delayed_list, will confuse pwq->nr_active
  1063. * management later on and cause stall. Make sure the work
  1064. * item is activated before grabbing.
  1065. */
  1066. if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
  1067. pwq_activate_delayed_work(work);
  1068. list_del_init(&work->entry);
  1069. pwq_dec_nr_in_flight(get_work_pwq(work), get_work_color(work));
  1070. /* work->data points to pwq iff queued, point to pool */
  1071. set_work_pool_and_keep_pending(work, pool->id);
  1072. spin_unlock(&pool->lock);
  1073. return 1;
  1074. }
  1075. spin_unlock(&pool->lock);
  1076. fail:
  1077. local_irq_restore(*flags);
  1078. if (work_is_canceling(work))
  1079. return -ENOENT;
  1080. cpu_relax();
  1081. return -EAGAIN;
  1082. }
  1083. /**
  1084. * insert_work - insert a work into a pool
  1085. * @pwq: pwq @work belongs to
  1086. * @work: work to insert
  1087. * @head: insertion point
  1088. * @extra_flags: extra WORK_STRUCT_* flags to set
  1089. *
  1090. * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
  1091. * work_struct flags.
  1092. *
  1093. * CONTEXT:
  1094. * spin_lock_irq(pool->lock).
  1095. */
  1096. static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
  1097. struct list_head *head, unsigned int extra_flags)
  1098. {
  1099. struct worker_pool *pool = pwq->pool;
  1100. /* we own @work, set data and link */
  1101. set_work_pwq(work, pwq, extra_flags);
  1102. list_add_tail(&work->entry, head);
  1103. get_pwq(pwq);
  1104. /*
  1105. * Ensure either wq_worker_sleeping() sees the above
  1106. * list_add_tail() or we see zero nr_running to avoid workers lying
  1107. * around lazily while there are works to be processed.
  1108. */
  1109. smp_mb();
  1110. if (__need_more_worker(pool))
  1111. wake_up_worker(pool);
  1112. }
  1113. /*
  1114. * Test whether @work is being queued from another work executing on the
  1115. * same workqueue.
  1116. */
  1117. static bool is_chained_work(struct workqueue_struct *wq)
  1118. {
  1119. struct worker *worker;
  1120. worker = current_wq_worker();
  1121. /*
  1122. * Return %true iff I'm a worker execuing a work item on @wq. If
  1123. * I'm @worker, it's safe to dereference it without locking.
  1124. */
  1125. return worker && worker->current_pwq->wq == wq;
  1126. }
  1127. static void __queue_work(int cpu, struct workqueue_struct *wq,
  1128. struct work_struct *work)
  1129. {
  1130. struct pool_workqueue *pwq;
  1131. struct worker_pool *last_pool;
  1132. struct list_head *worklist;
  1133. unsigned int work_flags;
  1134. unsigned int req_cpu = cpu;
  1135. /*
  1136. * While a work item is PENDING && off queue, a task trying to
  1137. * steal the PENDING will busy-loop waiting for it to either get
  1138. * queued or lose PENDING. Grabbing PENDING and queueing should
  1139. * happen with IRQ disabled.
  1140. */
  1141. WARN_ON_ONCE(!irqs_disabled());
  1142. debug_work_activate(work);
  1143. /* if dying, only works from the same workqueue are allowed */
  1144. if (unlikely(wq->flags & __WQ_DRAINING) &&
  1145. WARN_ON_ONCE(!is_chained_work(wq)))
  1146. return;
  1147. retry:
  1148. if (req_cpu == WORK_CPU_UNBOUND)
  1149. cpu = raw_smp_processor_id();
  1150. /* pwq which will be used unless @work is executing elsewhere */
  1151. if (!(wq->flags & WQ_UNBOUND))
  1152. pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
  1153. else
  1154. pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
  1155. /*
  1156. * If @work was previously on a different pool, it might still be
  1157. * running there, in which case the work needs to be queued on that
  1158. * pool to guarantee non-reentrancy.
  1159. */
  1160. last_pool = get_work_pool(work);
  1161. if (last_pool && last_pool != pwq->pool) {
  1162. struct worker *worker;
  1163. spin_lock(&last_pool->lock);
  1164. worker = find_worker_executing_work(last_pool, work);
  1165. if (worker && worker->current_pwq->wq == wq) {
  1166. pwq = worker->current_pwq;
  1167. } else {
  1168. /* meh... not running there, queue here */
  1169. spin_unlock(&last_pool->lock);
  1170. spin_lock(&pwq->pool->lock);
  1171. }
  1172. } else {
  1173. spin_lock(&pwq->pool->lock);
  1174. }
  1175. /*
  1176. * pwq is determined and locked. For unbound pools, we could have
  1177. * raced with pwq release and it could already be dead. If its
  1178. * refcnt is zero, repeat pwq selection. Note that pwqs never die
  1179. * without another pwq replacing it in the numa_pwq_tbl or while
  1180. * work items are executing on it, so the retrying is guaranteed to
  1181. * make forward-progress.
  1182. */
  1183. if (unlikely(!pwq->refcnt)) {
  1184. if (wq->flags & WQ_UNBOUND) {
  1185. spin_unlock(&pwq->pool->lock);
  1186. cpu_relax();
  1187. goto retry;
  1188. }
  1189. /* oops */
  1190. WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
  1191. wq->name, cpu);
  1192. }
  1193. /* pwq determined, queue */
  1194. trace_workqueue_queue_work(req_cpu, pwq, work);
  1195. if (WARN_ON(!list_empty(&work->entry))) {
  1196. spin_unlock(&pwq->pool->lock);
  1197. return;
  1198. }
  1199. pwq->nr_in_flight[pwq->work_color]++;
  1200. work_flags = work_color_to_flags(pwq->work_color);
  1201. if (likely(pwq->nr_active < pwq->max_active)) {
  1202. trace_workqueue_activate_work(work);
  1203. pwq->nr_active++;
  1204. worklist = &pwq->pool->worklist;
  1205. } else {
  1206. work_flags |= WORK_STRUCT_DELAYED;
  1207. worklist = &pwq->delayed_works;
  1208. }
  1209. insert_work(pwq, work, worklist, work_flags);
  1210. spin_unlock(&pwq->pool->lock);
  1211. }
  1212. /**
  1213. * queue_work_on - queue work on specific cpu
  1214. * @cpu: CPU number to execute work on
  1215. * @wq: workqueue to use
  1216. * @work: work to queue
  1217. *
  1218. * Returns %false if @work was already on a queue, %true otherwise.
  1219. *
  1220. * We queue the work to a specific CPU, the caller must ensure it
  1221. * can't go away.
  1222. */
  1223. bool queue_work_on(int cpu, struct workqueue_struct *wq,
  1224. struct work_struct *work)
  1225. {
  1226. bool ret = false;
  1227. unsigned long flags;
  1228. local_irq_save(flags);
  1229. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  1230. __queue_work(cpu, wq, work);
  1231. ret = true;
  1232. }
  1233. local_irq_restore(flags);
  1234. return ret;
  1235. }
  1236. EXPORT_SYMBOL_GPL(queue_work_on);
  1237. void delayed_work_timer_fn(unsigned long __data)
  1238. {
  1239. struct delayed_work *dwork = (struct delayed_work *)__data;
  1240. /* should have been called from irqsafe timer with irq already off */
  1241. __queue_work(dwork->cpu, dwork->wq, &dwork->work);
  1242. }
  1243. EXPORT_SYMBOL(delayed_work_timer_fn);
  1244. static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
  1245. struct delayed_work *dwork, unsigned long delay)
  1246. {
  1247. struct timer_list *timer = &dwork->timer;
  1248. struct work_struct *work = &dwork->work;
  1249. WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
  1250. timer->data != (unsigned long)dwork);
  1251. WARN_ON_ONCE(timer_pending(timer));
  1252. WARN_ON_ONCE(!list_empty(&work->entry));
  1253. /*
  1254. * If @delay is 0, queue @dwork->work immediately. This is for
  1255. * both optimization and correctness. The earliest @timer can
  1256. * expire is on the closest next tick and delayed_work users depend
  1257. * on that there's no such delay when @delay is 0.
  1258. */
  1259. if (!delay) {
  1260. __queue_work(cpu, wq, &dwork->work);
  1261. return;
  1262. }
  1263. timer_stats_timer_set_start_info(&dwork->timer);
  1264. dwork->wq = wq;
  1265. dwork->cpu = cpu;
  1266. timer->expires = jiffies + delay;
  1267. if (unlikely(cpu != WORK_CPU_UNBOUND))
  1268. add_timer_on(timer, cpu);
  1269. else
  1270. add_timer(timer);
  1271. }
  1272. /**
  1273. * queue_delayed_work_on - queue work on specific CPU after delay
  1274. * @cpu: CPU number to execute work on
  1275. * @wq: workqueue to use
  1276. * @dwork: work to queue
  1277. * @delay: number of jiffies to wait before queueing
  1278. *
  1279. * Returns %false if @work was already on a queue, %true otherwise. If
  1280. * @delay is zero and @dwork is idle, it will be scheduled for immediate
  1281. * execution.
  1282. */
  1283. bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
  1284. struct delayed_work *dwork, unsigned long delay)
  1285. {
  1286. struct work_struct *work = &dwork->work;
  1287. bool ret = false;
  1288. unsigned long flags;
  1289. /* read the comment in __queue_work() */
  1290. local_irq_save(flags);
  1291. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  1292. __queue_delayed_work(cpu, wq, dwork, delay);
  1293. ret = true;
  1294. }
  1295. local_irq_restore(flags);
  1296. return ret;
  1297. }
  1298. EXPORT_SYMBOL_GPL(queue_delayed_work_on);
  1299. /**
  1300. * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
  1301. * @cpu: CPU number to execute work on
  1302. * @wq: workqueue to use
  1303. * @dwork: work to queue
  1304. * @delay: number of jiffies to wait before queueing
  1305. *
  1306. * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
  1307. * modify @dwork's timer so that it expires after @delay. If @delay is
  1308. * zero, @work is guaranteed to be scheduled immediately regardless of its
  1309. * current state.
  1310. *
  1311. * Returns %false if @dwork was idle and queued, %true if @dwork was
  1312. * pending and its timer was modified.
  1313. *
  1314. * This function is safe to call from any context including IRQ handler.
  1315. * See try_to_grab_pending() for details.
  1316. */
  1317. bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
  1318. struct delayed_work *dwork, unsigned long delay)
  1319. {
  1320. unsigned long flags;
  1321. int ret;
  1322. do {
  1323. ret = try_to_grab_pending(&dwork->work, true, &flags);
  1324. } while (unlikely(ret == -EAGAIN));
  1325. if (likely(ret >= 0)) {
  1326. __queue_delayed_work(cpu, wq, dwork, delay);
  1327. local_irq_restore(flags);
  1328. }
  1329. /* -ENOENT from try_to_grab_pending() becomes %true */
  1330. return ret;
  1331. }
  1332. EXPORT_SYMBOL_GPL(mod_delayed_work_on);
  1333. /**
  1334. * worker_enter_idle - enter idle state
  1335. * @worker: worker which is entering idle state
  1336. *
  1337. * @worker is entering idle state. Update stats and idle timer if
  1338. * necessary.
  1339. *
  1340. * LOCKING:
  1341. * spin_lock_irq(pool->lock).
  1342. */
  1343. static void worker_enter_idle(struct worker *worker)
  1344. {
  1345. struct worker_pool *pool = worker->pool;
  1346. if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
  1347. WARN_ON_ONCE(!list_empty(&worker->entry) &&
  1348. (worker->hentry.next || worker->hentry.pprev)))
  1349. return;
  1350. /* can't use worker_set_flags(), also called from start_worker() */
  1351. worker->flags |= WORKER_IDLE;
  1352. pool->nr_idle++;
  1353. worker->last_active = jiffies;
  1354. /* idle_list is LIFO */
  1355. list_add(&worker->entry, &pool->idle_list);
  1356. if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
  1357. mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
  1358. /*
  1359. * Sanity check nr_running. Because wq_unbind_fn() releases
  1360. * pool->lock between setting %WORKER_UNBOUND and zapping
  1361. * nr_running, the warning may trigger spuriously. Check iff
  1362. * unbind is not in progress.
  1363. */
  1364. WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
  1365. pool->nr_workers == pool->nr_idle &&
  1366. atomic_read(&pool->nr_running));
  1367. }
  1368. /**
  1369. * worker_leave_idle - leave idle state
  1370. * @worker: worker which is leaving idle state
  1371. *
  1372. * @worker is leaving idle state. Update stats.
  1373. *
  1374. * LOCKING:
  1375. * spin_lock_irq(pool->lock).
  1376. */
  1377. static void worker_leave_idle(struct worker *worker)
  1378. {
  1379. struct worker_pool *pool = worker->pool;
  1380. if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
  1381. return;
  1382. worker_clr_flags(worker, WORKER_IDLE);
  1383. pool->nr_idle--;
  1384. list_del_init(&worker->entry);
  1385. }
  1386. /**
  1387. * worker_maybe_bind_and_lock - try to bind %current to worker_pool and lock it
  1388. * @pool: target worker_pool
  1389. *
  1390. * Bind %current to the cpu of @pool if it is associated and lock @pool.
  1391. *
  1392. * Works which are scheduled while the cpu is online must at least be
  1393. * scheduled to a worker which is bound to the cpu so that if they are
  1394. * flushed from cpu callbacks while cpu is going down, they are
  1395. * guaranteed to execute on the cpu.
  1396. *
  1397. * This function is to be used by unbound workers and rescuers to bind
  1398. * themselves to the target cpu and may race with cpu going down or
  1399. * coming online. kthread_bind() can't be used because it may put the
  1400. * worker to already dead cpu and set_cpus_allowed_ptr() can't be used
  1401. * verbatim as it's best effort and blocking and pool may be
  1402. * [dis]associated in the meantime.
  1403. *
  1404. * This function tries set_cpus_allowed() and locks pool and verifies the
  1405. * binding against %POOL_DISASSOCIATED which is set during
  1406. * %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker
  1407. * enters idle state or fetches works without dropping lock, it can
  1408. * guarantee the scheduling requirement described in the first paragraph.
  1409. *
  1410. * CONTEXT:
  1411. * Might sleep. Called without any lock but returns with pool->lock
  1412. * held.
  1413. *
  1414. * RETURNS:
  1415. * %true if the associated pool is online (@worker is successfully
  1416. * bound), %false if offline.
  1417. */
  1418. static bool worker_maybe_bind_and_lock(struct worker_pool *pool)
  1419. __acquires(&pool->lock)
  1420. {
  1421. while (true) {
  1422. /*
  1423. * The following call may fail, succeed or succeed
  1424. * without actually migrating the task to the cpu if
  1425. * it races with cpu hotunplug operation. Verify
  1426. * against POOL_DISASSOCIATED.
  1427. */
  1428. if (!(pool->flags & POOL_DISASSOCIATED))
  1429. set_cpus_allowed_ptr(current, pool->attrs->cpumask);
  1430. spin_lock_irq(&pool->lock);
  1431. if (pool->flags & POOL_DISASSOCIATED)
  1432. return false;
  1433. if (task_cpu(current) == pool->cpu &&
  1434. cpumask_equal(&current->cpus_allowed, pool->attrs->cpumask))
  1435. return true;
  1436. spin_unlock_irq(&pool->lock);
  1437. /*
  1438. * We've raced with CPU hot[un]plug. Give it a breather
  1439. * and retry migration. cond_resched() is required here;
  1440. * otherwise, we might deadlock against cpu_stop trying to
  1441. * bring down the CPU on non-preemptive kernel.
  1442. */
  1443. cpu_relax();
  1444. cond_resched();
  1445. }
  1446. }
  1447. static struct worker *alloc_worker(void)
  1448. {
  1449. struct worker *worker;
  1450. worker = kzalloc(sizeof(*worker), GFP_KERNEL);
  1451. if (worker) {
  1452. INIT_LIST_HEAD(&worker->entry);
  1453. INIT_LIST_HEAD(&worker->scheduled);
  1454. /* on creation a worker is in !idle && prep state */
  1455. worker->flags = WORKER_PREP;
  1456. }
  1457. return worker;
  1458. }
  1459. /**
  1460. * create_worker - create a new workqueue worker
  1461. * @pool: pool the new worker will belong to
  1462. *
  1463. * Create a new worker which is bound to @pool. The returned worker
  1464. * can be started by calling start_worker() or destroyed using
  1465. * destroy_worker().
  1466. *
  1467. * CONTEXT:
  1468. * Might sleep. Does GFP_KERNEL allocations.
  1469. *
  1470. * RETURNS:
  1471. * Pointer to the newly created worker.
  1472. */
  1473. static struct worker *create_worker(struct worker_pool *pool)
  1474. {
  1475. struct worker *worker = NULL;
  1476. int id = -1;
  1477. char id_buf[16];
  1478. lockdep_assert_held(&pool->manager_mutex);
  1479. /*
  1480. * ID is needed to determine kthread name. Allocate ID first
  1481. * without installing the pointer.
  1482. */
  1483. idr_preload(GFP_KERNEL);
  1484. spin_lock_irq(&pool->lock);
  1485. id = idr_alloc(&pool->worker_idr, NULL, 0, 0, GFP_NOWAIT);
  1486. spin_unlock_irq(&pool->lock);
  1487. idr_preload_end();
  1488. if (id < 0)
  1489. goto fail;
  1490. worker = alloc_worker();
  1491. if (!worker)
  1492. goto fail;
  1493. worker->pool = pool;
  1494. worker->id = id;
  1495. if (pool->cpu >= 0)
  1496. snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
  1497. pool->attrs->nice < 0 ? "H" : "");
  1498. else
  1499. snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
  1500. worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
  1501. "kworker/%s", id_buf);
  1502. if (IS_ERR(worker->task))
  1503. goto fail;
  1504. /*
  1505. * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
  1506. * online CPUs. It'll be re-applied when any of the CPUs come up.
  1507. */
  1508. set_user_nice(worker->task, pool->attrs->nice);
  1509. set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
  1510. /* prevent userland from meddling with cpumask of workqueue workers */
  1511. worker->task->flags |= PF_NO_SETAFFINITY;
  1512. /*
  1513. * The caller is responsible for ensuring %POOL_DISASSOCIATED
  1514. * remains stable across this function. See the comments above the
  1515. * flag definition for details.
  1516. */
  1517. if (pool->flags & POOL_DISASSOCIATED)
  1518. worker->flags |= WORKER_UNBOUND;
  1519. /* successful, commit the pointer to idr */
  1520. spin_lock_irq(&pool->lock);
  1521. idr_replace(&pool->worker_idr, worker, worker->id);
  1522. spin_unlock_irq(&pool->lock);
  1523. return worker;
  1524. fail:
  1525. if (id >= 0) {
  1526. spin_lock_irq(&pool->lock);
  1527. idr_remove(&pool->worker_idr, id);
  1528. spin_unlock_irq(&pool->lock);
  1529. }
  1530. kfree(worker);
  1531. return NULL;
  1532. }
  1533. /**
  1534. * start_worker - start a newly created worker
  1535. * @worker: worker to start
  1536. *
  1537. * Make the pool aware of @worker and start it.
  1538. *
  1539. * CONTEXT:
  1540. * spin_lock_irq(pool->lock).
  1541. */
  1542. static void start_worker(struct worker *worker)
  1543. {
  1544. worker->flags |= WORKER_STARTED;
  1545. worker->pool->nr_workers++;
  1546. worker_enter_idle(worker);
  1547. wake_up_process(worker->task);
  1548. }
  1549. /**
  1550. * create_and_start_worker - create and start a worker for a pool
  1551. * @pool: the target pool
  1552. *
  1553. * Grab the managership of @pool and create and start a new worker for it.
  1554. */
  1555. static int create_and_start_worker(struct worker_pool *pool)
  1556. {
  1557. struct worker *worker;
  1558. mutex_lock(&pool->manager_mutex);
  1559. worker = create_worker(pool);
  1560. if (worker) {
  1561. spin_lock_irq(&pool->lock);
  1562. start_worker(worker);
  1563. spin_unlock_irq(&pool->lock);
  1564. }
  1565. mutex_unlock(&pool->manager_mutex);
  1566. return worker ? 0 : -ENOMEM;
  1567. }
  1568. /**
  1569. * destroy_worker - destroy a workqueue worker
  1570. * @worker: worker to be destroyed
  1571. *
  1572. * Destroy @worker and adjust @pool stats accordingly.
  1573. *
  1574. * CONTEXT:
  1575. * spin_lock_irq(pool->lock) which is released and regrabbed.
  1576. */
  1577. static void destroy_worker(struct worker *worker)
  1578. {
  1579. struct worker_pool *pool = worker->pool;
  1580. lockdep_assert_held(&pool->manager_mutex);
  1581. lockdep_assert_held(&pool->lock);
  1582. /* sanity check frenzy */
  1583. if (WARN_ON(worker->current_work) ||
  1584. WARN_ON(!list_empty(&worker->scheduled)))
  1585. return;
  1586. if (worker->flags & WORKER_STARTED)
  1587. pool->nr_workers--;
  1588. if (worker->flags & WORKER_IDLE)
  1589. pool->nr_idle--;
  1590. list_del_init(&worker->entry);
  1591. worker->flags |= WORKER_DIE;
  1592. idr_remove(&pool->worker_idr, worker->id);
  1593. spin_unlock_irq(&pool->lock);
  1594. kthread_stop(worker->task);
  1595. kfree(worker);
  1596. spin_lock_irq(&pool->lock);
  1597. }
  1598. static void idle_worker_timeout(unsigned long __pool)
  1599. {
  1600. struct worker_pool *pool = (void *)__pool;
  1601. spin_lock_irq(&pool->lock);
  1602. if (too_many_workers(pool)) {
  1603. struct worker *worker;
  1604. unsigned long expires;
  1605. /* idle_list is kept in LIFO order, check the last one */
  1606. worker = list_entry(pool->idle_list.prev, struct worker, entry);
  1607. expires = worker->last_active + IDLE_WORKER_TIMEOUT;
  1608. if (time_before(jiffies, expires))
  1609. mod_timer(&pool->idle_timer, expires);
  1610. else {
  1611. /* it's been idle for too long, wake up manager */
  1612. pool->flags |= POOL_MANAGE_WORKERS;
  1613. wake_up_worker(pool);
  1614. }
  1615. }
  1616. spin_unlock_irq(&pool->lock);
  1617. }
  1618. static void send_mayday(struct work_struct *work)
  1619. {
  1620. struct pool_workqueue *pwq = get_work_pwq(work);
  1621. struct workqueue_struct *wq = pwq->wq;
  1622. lockdep_assert_held(&wq_mayday_lock);
  1623. if (!wq->rescuer)
  1624. return;
  1625. /* mayday mayday mayday */
  1626. if (list_empty(&pwq->mayday_node)) {
  1627. list_add_tail(&pwq->mayday_node, &wq->maydays);
  1628. wake_up_process(wq->rescuer->task);
  1629. }
  1630. }
  1631. static void pool_mayday_timeout(unsigned long __pool)
  1632. {
  1633. struct worker_pool *pool = (void *)__pool;
  1634. struct work_struct *work;
  1635. spin_lock_irq(&wq_mayday_lock); /* for wq->maydays */
  1636. spin_lock(&pool->lock);
  1637. if (need_to_create_worker(pool)) {
  1638. /*
  1639. * We've been trying to create a new worker but
  1640. * haven't been successful. We might be hitting an
  1641. * allocation deadlock. Send distress signals to
  1642. * rescuers.
  1643. */
  1644. list_for_each_entry(work, &pool->worklist, entry)
  1645. send_mayday(work);
  1646. }
  1647. spin_unlock(&pool->lock);
  1648. spin_unlock_irq(&wq_mayday_lock);
  1649. mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
  1650. }
  1651. /**
  1652. * maybe_create_worker - create a new worker if necessary
  1653. * @pool: pool to create a new worker for
  1654. *
  1655. * Create a new worker for @pool if necessary. @pool is guaranteed to
  1656. * have at least one idle worker on return from this function. If
  1657. * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
  1658. * sent to all rescuers with works scheduled on @pool to resolve
  1659. * possible allocation deadlock.
  1660. *
  1661. * On return, need_to_create_worker() is guaranteed to be %false and
  1662. * may_start_working() %true.
  1663. *
  1664. * LOCKING:
  1665. * spin_lock_irq(pool->lock) which may be released and regrabbed
  1666. * multiple times. Does GFP_KERNEL allocations. Called only from
  1667. * manager.
  1668. *
  1669. * RETURNS:
  1670. * %false if no action was taken and pool->lock stayed locked, %true
  1671. * otherwise.
  1672. */
  1673. static bool maybe_create_worker(struct worker_pool *pool)
  1674. __releases(&pool->lock)
  1675. __acquires(&pool->lock)
  1676. {
  1677. if (!need_to_create_worker(pool))
  1678. return false;
  1679. restart:
  1680. spin_unlock_irq(&pool->lock);
  1681. /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
  1682. mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
  1683. while (true) {
  1684. struct worker *worker;
  1685. worker = create_worker(pool);
  1686. if (worker) {
  1687. del_timer_sync(&pool->mayday_timer);
  1688. spin_lock_irq(&pool->lock);
  1689. start_worker(worker);
  1690. if (WARN_ON_ONCE(need_to_create_worker(pool)))
  1691. goto restart;
  1692. return true;
  1693. }
  1694. if (!need_to_create_worker(pool))
  1695. break;
  1696. __set_current_state(TASK_INTERRUPTIBLE);
  1697. schedule_timeout(CREATE_COOLDOWN);
  1698. if (!need_to_create_worker(pool))
  1699. break;
  1700. }
  1701. del_timer_sync(&pool->mayday_timer);
  1702. spin_lock_irq(&pool->lock);
  1703. if (need_to_create_worker(pool))
  1704. goto restart;
  1705. return true;
  1706. }
  1707. /**
  1708. * maybe_destroy_worker - destroy workers which have been idle for a while
  1709. * @pool: pool to destroy workers for
  1710. *
  1711. * Destroy @pool workers which have been idle for longer than
  1712. * IDLE_WORKER_TIMEOUT.
  1713. *
  1714. * LOCKING:
  1715. * spin_lock_irq(pool->lock) which may be released and regrabbed
  1716. * multiple times. Called only from manager.
  1717. *
  1718. * RETURNS:
  1719. * %false if no action was taken and pool->lock stayed locked, %true
  1720. * otherwise.
  1721. */
  1722. static bool maybe_destroy_workers(struct worker_pool *pool)
  1723. {
  1724. bool ret = false;
  1725. while (too_many_workers(pool)) {
  1726. struct worker *worker;
  1727. unsigned long expires;
  1728. worker = list_entry(pool->idle_list.prev, struct worker, entry);
  1729. expires = worker->last_active + IDLE_WORKER_TIMEOUT;
  1730. if (time_before(jiffies, expires)) {
  1731. mod_timer(&pool->idle_timer, expires);
  1732. break;
  1733. }
  1734. destroy_worker(worker);
  1735. ret = true;
  1736. }
  1737. return ret;
  1738. }
  1739. /**
  1740. * manage_workers - manage worker pool
  1741. * @worker: self
  1742. *
  1743. * Assume the manager role and manage the worker pool @worker belongs
  1744. * to. At any given time, there can be only zero or one manager per
  1745. * pool. The exclusion is handled automatically by this function.
  1746. *
  1747. * The caller can safely start processing works on false return. On
  1748. * true return, it's guaranteed that need_to_create_worker() is false
  1749. * and may_start_working() is true.
  1750. *
  1751. * CONTEXT:
  1752. * spin_lock_irq(pool->lock) which may be released and regrabbed
  1753. * multiple times. Does GFP_KERNEL allocations.
  1754. *
  1755. * RETURNS:
  1756. * spin_lock_irq(pool->lock) which may be released and regrabbed
  1757. * multiple times. Does GFP_KERNEL allocations.
  1758. */
  1759. static bool manage_workers(struct worker *worker)
  1760. {
  1761. struct worker_pool *pool = worker->pool;
  1762. bool ret = false;
  1763. /*
  1764. * Managership is governed by two mutexes - manager_arb and
  1765. * manager_mutex. manager_arb handles arbitration of manager role.
  1766. * Anyone who successfully grabs manager_arb wins the arbitration
  1767. * and becomes the manager. mutex_trylock() on pool->manager_arb
  1768. * failure while holding pool->lock reliably indicates that someone
  1769. * else is managing the pool and the worker which failed trylock
  1770. * can proceed to executing work items. This means that anyone
  1771. * grabbing manager_arb is responsible for actually performing
  1772. * manager duties. If manager_arb is grabbed and released without
  1773. * actual management, the pool may stall indefinitely.
  1774. *
  1775. * manager_mutex is used for exclusion of actual management
  1776. * operations. The holder of manager_mutex can be sure that none
  1777. * of management operations, including creation and destruction of
  1778. * workers, won't take place until the mutex is released. Because
  1779. * manager_mutex doesn't interfere with manager role arbitration,
  1780. * it is guaranteed that the pool's management, while may be
  1781. * delayed, won't be disturbed by someone else grabbing
  1782. * manager_mutex.
  1783. */
  1784. if (!mutex_trylock(&pool->manager_arb))
  1785. return ret;
  1786. /*
  1787. * With manager arbitration won, manager_mutex would be free in
  1788. * most cases. trylock first without dropping @pool->lock.
  1789. */
  1790. if (unlikely(!mutex_trylock(&pool->manager_mutex))) {
  1791. spin_unlock_irq(&pool->lock);
  1792. mutex_lock(&pool->manager_mutex);
  1793. ret = true;
  1794. }
  1795. pool->flags &= ~POOL_MANAGE_WORKERS;
  1796. /*
  1797. * Destroy and then create so that may_start_working() is true
  1798. * on return.
  1799. */
  1800. ret |= maybe_destroy_workers(pool);
  1801. ret |= maybe_create_worker(pool);
  1802. mutex_unlock(&pool->manager_mutex);
  1803. mutex_unlock(&pool->manager_arb);
  1804. return ret;
  1805. }
  1806. /**
  1807. * process_one_work - process single work
  1808. * @worker: self
  1809. * @work: work to process
  1810. *
  1811. * Process @work. This function contains all the logics necessary to
  1812. * process a single work including synchronization against and
  1813. * interaction with other workers on the same cpu, queueing and
  1814. * flushing. As long as context requirement is met, any worker can
  1815. * call this function to process a work.
  1816. *
  1817. * CONTEXT:
  1818. * spin_lock_irq(pool->lock) which is released and regrabbed.
  1819. */
  1820. static void process_one_work(struct worker *worker, struct work_struct *work)
  1821. __releases(&pool->lock)
  1822. __acquires(&pool->lock)
  1823. {
  1824. struct pool_workqueue *pwq = get_work_pwq(work);
  1825. struct worker_pool *pool = worker->pool;
  1826. bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
  1827. int work_color;
  1828. struct worker *collision;
  1829. #ifdef CONFIG_LOCKDEP
  1830. /*
  1831. * It is permissible to free the struct work_struct from
  1832. * inside the function that is called from it, this we need to
  1833. * take into account for lockdep too. To avoid bogus "held
  1834. * lock freed" warnings as well as problems when looking into
  1835. * work->lockdep_map, make a copy and use that here.
  1836. */
  1837. struct lockdep_map lockdep_map;
  1838. lockdep_copy_map(&lockdep_map, &work->lockdep_map);
  1839. #endif
  1840. /*
  1841. * Ensure we're on the correct CPU. DISASSOCIATED test is
  1842. * necessary to avoid spurious warnings from rescuers servicing the
  1843. * unbound or a disassociated pool.
  1844. */
  1845. WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
  1846. !(pool->flags & POOL_DISASSOCIATED) &&
  1847. raw_smp_processor_id() != pool->cpu);
  1848. /*
  1849. * A single work shouldn't be executed concurrently by
  1850. * multiple workers on a single cpu. Check whether anyone is
  1851. * already processing the work. If so, defer the work to the
  1852. * currently executing one.
  1853. */
  1854. collision = find_worker_executing_work(pool, work);
  1855. if (unlikely(collision)) {
  1856. move_linked_works(work, &collision->scheduled, NULL);
  1857. return;
  1858. }
  1859. /* claim and dequeue */
  1860. debug_work_deactivate(work);
  1861. hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
  1862. worker->current_work = work;
  1863. worker->current_func = work->func;
  1864. worker->current_pwq = pwq;
  1865. work_color = get_work_color(work);
  1866. list_del_init(&work->entry);
  1867. /*
  1868. * CPU intensive works don't participate in concurrency
  1869. * management. They're the scheduler's responsibility.
  1870. */
  1871. if (unlikely(cpu_intensive))
  1872. worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
  1873. /*
  1874. * Unbound pool isn't concurrency managed and work items should be
  1875. * executed ASAP. Wake up another worker if necessary.
  1876. */
  1877. if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
  1878. wake_up_worker(pool);
  1879. /*
  1880. * Record the last pool and clear PENDING which should be the last
  1881. * update to @work. Also, do this inside @pool->lock so that
  1882. * PENDING and queued state changes happen together while IRQ is
  1883. * disabled.
  1884. */
  1885. set_work_pool_and_clear_pending(work, pool->id);
  1886. spin_unlock_irq(&pool->lock);
  1887. lock_map_acquire_read(&pwq->wq->lockdep_map);
  1888. lock_map_acquire(&lockdep_map);
  1889. trace_workqueue_execute_start(work);
  1890. worker->current_func(work);
  1891. /*
  1892. * While we must be careful to not use "work" after this, the trace
  1893. * point will only record its address.
  1894. */
  1895. trace_workqueue_execute_end(work);
  1896. lock_map_release(&lockdep_map);
  1897. lock_map_release(&pwq->wq->lockdep_map);
  1898. if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
  1899. pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
  1900. " last function: %pf\n",
  1901. current->comm, preempt_count(), task_pid_nr(current),
  1902. worker->current_func);
  1903. debug_show_held_locks(current);
  1904. dump_stack();
  1905. }
  1906. spin_lock_irq(&pool->lock);
  1907. /* clear cpu intensive status */
  1908. if (unlikely(cpu_intensive))
  1909. worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
  1910. /* we're done with it, release */
  1911. hash_del(&worker->hentry);
  1912. worker->current_work = NULL;
  1913. worker->current_func = NULL;
  1914. worker->current_pwq = NULL;
  1915. pwq_dec_nr_in_flight(pwq, work_color);
  1916. }
  1917. /**
  1918. * process_scheduled_works - process scheduled works
  1919. * @worker: self
  1920. *
  1921. * Process all scheduled works. Please note that the scheduled list
  1922. * may change while processing a work, so this function repeatedly
  1923. * fetches a work from the top and executes it.
  1924. *
  1925. * CONTEXT:
  1926. * spin_lock_irq(pool->lock) which may be released and regrabbed
  1927. * multiple times.
  1928. */
  1929. static void process_scheduled_works(struct worker *worker)
  1930. {
  1931. while (!list_empty(&worker->scheduled)) {
  1932. struct work_struct *work = list_first_entry(&worker->scheduled,
  1933. struct work_struct, entry);
  1934. process_one_work(worker, work);
  1935. }
  1936. }
  1937. /**
  1938. * worker_thread - the worker thread function
  1939. * @__worker: self
  1940. *
  1941. * The worker thread function. All workers belong to a worker_pool -
  1942. * either a per-cpu one or dynamic unbound one. These workers process all
  1943. * work items regardless of their specific target workqueue. The only
  1944. * exception is work items which belong to workqueues with a rescuer which
  1945. * will be explained in rescuer_thread().
  1946. */
  1947. static int worker_thread(void *__worker)
  1948. {
  1949. struct worker *worker = __worker;
  1950. struct worker_pool *pool = worker->pool;
  1951. /* tell the scheduler that this is a workqueue worker */
  1952. worker->task->flags |= PF_WQ_WORKER;
  1953. woke_up:
  1954. spin_lock_irq(&pool->lock);
  1955. /* am I supposed to die? */
  1956. if (unlikely(worker->flags & WORKER_DIE)) {
  1957. spin_unlock_irq(&pool->lock);
  1958. WARN_ON_ONCE(!list_empty(&worker->entry));
  1959. worker->task->flags &= ~PF_WQ_WORKER;
  1960. return 0;
  1961. }
  1962. worker_leave_idle(worker);
  1963. recheck:
  1964. /* no more worker necessary? */
  1965. if (!need_more_worker(pool))
  1966. goto sleep;
  1967. /* do we need to manage? */
  1968. if (unlikely(!may_start_working(pool)) && manage_workers(worker))
  1969. goto recheck;
  1970. /*
  1971. * ->scheduled list can only be filled while a worker is
  1972. * preparing to process a work or actually processing it.
  1973. * Make sure nobody diddled with it while I was sleeping.
  1974. */
  1975. WARN_ON_ONCE(!list_empty(&worker->scheduled));
  1976. /*
  1977. * Finish PREP stage. We're guaranteed to have at least one idle
  1978. * worker or that someone else has already assumed the manager
  1979. * role. This is where @worker starts participating in concurrency
  1980. * management if applicable and concurrency management is restored
  1981. * after being rebound. See rebind_workers() for details.
  1982. */
  1983. worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
  1984. do {
  1985. struct work_struct *work =
  1986. list_first_entry(&pool->worklist,
  1987. struct work_struct, entry);
  1988. if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
  1989. /* optimization path, not strictly necessary */
  1990. process_one_work(worker, work);
  1991. if (unlikely(!list_empty(&worker->scheduled)))
  1992. process_scheduled_works(worker);
  1993. } else {
  1994. move_linked_works(work, &worker->scheduled, NULL);
  1995. process_scheduled_works(worker);
  1996. }
  1997. } while (keep_working(pool));
  1998. worker_set_flags(worker, WORKER_PREP, false);
  1999. sleep:
  2000. if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
  2001. goto recheck;
  2002. /*
  2003. * pool->lock is held and there's no work to process and no need to
  2004. * manage, sleep. Workers are woken up only while holding
  2005. * pool->lock or from local cpu, so setting the current state
  2006. * before releasing pool->lock is enough to prevent losing any
  2007. * event.
  2008. */
  2009. worker_enter_idle(worker);
  2010. __set_current_state(TASK_INTERRUPTIBLE);
  2011. spin_unlock_irq(&pool->lock);
  2012. schedule();
  2013. goto woke_up;
  2014. }
  2015. /**
  2016. * rescuer_thread - the rescuer thread function
  2017. * @__rescuer: self
  2018. *
  2019. * Workqueue rescuer thread function. There's one rescuer for each
  2020. * workqueue which has WQ_MEM_RECLAIM set.
  2021. *
  2022. * Regular work processing on a pool may block trying to create a new
  2023. * worker which uses GFP_KERNEL allocation which has slight chance of
  2024. * developing into deadlock if some works currently on the same queue
  2025. * need to be processed to satisfy the GFP_KERNEL allocation. This is
  2026. * the problem rescuer solves.
  2027. *
  2028. * When such condition is possible, the pool summons rescuers of all
  2029. * workqueues which have works queued on the pool and let them process
  2030. * those works so that forward progress can be guaranteed.
  2031. *
  2032. * This should happen rarely.
  2033. */
  2034. static int rescuer_thread(void *__rescuer)
  2035. {
  2036. struct worker *rescuer = __rescuer;
  2037. struct workqueue_struct *wq = rescuer->rescue_wq;
  2038. struct list_head *scheduled = &rescuer->scheduled;
  2039. set_user_nice(current, RESCUER_NICE_LEVEL);
  2040. /*
  2041. * Mark rescuer as worker too. As WORKER_PREP is never cleared, it
  2042. * doesn't participate in concurrency management.
  2043. */
  2044. rescuer->task->flags |= PF_WQ_WORKER;
  2045. repeat:
  2046. set_current_state(TASK_INTERRUPTIBLE);
  2047. if (kthread_should_stop()) {
  2048. __set_current_state(TASK_RUNNING);
  2049. rescuer->task->flags &= ~PF_WQ_WORKER;
  2050. return 0;
  2051. }
  2052. /* see whether any pwq is asking for help */
  2053. spin_lock_irq(&wq_mayday_lock);
  2054. while (!list_empty(&wq->maydays)) {
  2055. struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
  2056. struct pool_workqueue, mayday_node);
  2057. struct worker_pool *pool = pwq->pool;
  2058. struct work_struct *work, *n;
  2059. __set_current_state(TASK_RUNNING);
  2060. list_del_init(&pwq->mayday_node);
  2061. spin_unlock_irq(&wq_mayday_lock);
  2062. /* migrate to the target cpu if possible */
  2063. worker_maybe_bind_and_lock(pool);
  2064. rescuer->pool = pool;
  2065. /*
  2066. * Slurp in all works issued via this workqueue and
  2067. * process'em.
  2068. */
  2069. WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
  2070. list_for_each_entry_safe(work, n, &pool->worklist, entry)
  2071. if (get_work_pwq(work) == pwq)
  2072. move_linked_works(work, scheduled, &n);
  2073. process_scheduled_works(rescuer);
  2074. /*
  2075. * Leave this pool. If keep_working() is %true, notify a
  2076. * regular worker; otherwise, we end up with 0 concurrency
  2077. * and stalling the execution.
  2078. */
  2079. if (keep_working(pool))
  2080. wake_up_worker(pool);
  2081. rescuer->pool = NULL;
  2082. spin_unlock(&pool->lock);
  2083. spin_lock(&wq_mayday_lock);
  2084. }
  2085. spin_unlock_irq(&wq_mayday_lock);
  2086. /* rescuers should never participate in concurrency management */
  2087. WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
  2088. schedule();
  2089. goto repeat;
  2090. }
  2091. struct wq_barrier {
  2092. struct work_struct work;
  2093. struct completion done;
  2094. };
  2095. static void wq_barrier_func(struct work_struct *work)
  2096. {
  2097. struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
  2098. complete(&barr->done);
  2099. }
  2100. /**
  2101. * insert_wq_barrier - insert a barrier work
  2102. * @pwq: pwq to insert barrier into
  2103. * @barr: wq_barrier to insert
  2104. * @target: target work to attach @barr to
  2105. * @worker: worker currently executing @target, NULL if @target is not executing
  2106. *
  2107. * @barr is linked to @target such that @barr is completed only after
  2108. * @target finishes execution. Please note that the ordering
  2109. * guarantee is observed only with respect to @target and on the local
  2110. * cpu.
  2111. *
  2112. * Currently, a queued barrier can't be canceled. This is because
  2113. * try_to_grab_pending() can't determine whether the work to be
  2114. * grabbed is at the head of the queue and thus can't clear LINKED
  2115. * flag of the previous work while there must be a valid next work
  2116. * after a work with LINKED flag set.
  2117. *
  2118. * Note that when @worker is non-NULL, @target may be modified
  2119. * underneath us, so we can't reliably determine pwq from @target.
  2120. *
  2121. * CONTEXT:
  2122. * spin_lock_irq(pool->lock).
  2123. */
  2124. static void insert_wq_barrier(struct pool_workqueue *pwq,
  2125. struct wq_barrier *barr,
  2126. struct work_struct *target, struct worker *worker)
  2127. {
  2128. struct list_head *head;
  2129. unsigned int linked = 0;
  2130. /*
  2131. * debugobject calls are safe here even with pool->lock locked
  2132. * as we know for sure that this will not trigger any of the
  2133. * checks and call back into the fixup functions where we
  2134. * might deadlock.
  2135. */
  2136. INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
  2137. __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
  2138. init_completion(&barr->done);
  2139. /*
  2140. * If @target is currently being executed, schedule the
  2141. * barrier to the worker; otherwise, put it after @target.
  2142. */
  2143. if (worker)
  2144. head = worker->scheduled.next;
  2145. else {
  2146. unsigned long *bits = work_data_bits(target);
  2147. head = target->entry.next;
  2148. /* there can already be other linked works, inherit and set */
  2149. linked = *bits & WORK_STRUCT_LINKED;
  2150. __set_bit(WORK_STRUCT_LINKED_BIT, bits);
  2151. }
  2152. debug_work_activate(&barr->work);
  2153. insert_work(pwq, &barr->work, head,
  2154. work_color_to_flags(WORK_NO_COLOR) | linked);
  2155. }
  2156. /**
  2157. * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
  2158. * @wq: workqueue being flushed
  2159. * @flush_color: new flush color, < 0 for no-op
  2160. * @work_color: new work color, < 0 for no-op
  2161. *
  2162. * Prepare pwqs for workqueue flushing.
  2163. *
  2164. * If @flush_color is non-negative, flush_color on all pwqs should be
  2165. * -1. If no pwq has in-flight commands at the specified color, all
  2166. * pwq->flush_color's stay at -1 and %false is returned. If any pwq
  2167. * has in flight commands, its pwq->flush_color is set to
  2168. * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
  2169. * wakeup logic is armed and %true is returned.
  2170. *
  2171. * The caller should have initialized @wq->first_flusher prior to
  2172. * calling this function with non-negative @flush_color. If
  2173. * @flush_color is negative, no flush color update is done and %false
  2174. * is returned.
  2175. *
  2176. * If @work_color is non-negative, all pwqs should have the same
  2177. * work_color which is previous to @work_color and all will be
  2178. * advanced to @work_color.
  2179. *
  2180. * CONTEXT:
  2181. * mutex_lock(wq->mutex).
  2182. *
  2183. * RETURNS:
  2184. * %true if @flush_color >= 0 and there's something to flush. %false
  2185. * otherwise.
  2186. */
  2187. static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
  2188. int flush_color, int work_color)
  2189. {
  2190. bool wait = false;
  2191. struct pool_workqueue *pwq;
  2192. if (flush_color >= 0) {
  2193. WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
  2194. atomic_set(&wq->nr_pwqs_to_flush, 1);
  2195. }
  2196. for_each_pwq(pwq, wq) {
  2197. struct worker_pool *pool = pwq->pool;
  2198. spin_lock_irq(&pool->lock);
  2199. if (flush_color >= 0) {
  2200. WARN_ON_ONCE(pwq->flush_color != -1);
  2201. if (pwq->nr_in_flight[flush_color]) {
  2202. pwq->flush_color = flush_color;
  2203. atomic_inc(&wq->nr_pwqs_to_flush);
  2204. wait = true;
  2205. }
  2206. }
  2207. if (work_color >= 0) {
  2208. WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
  2209. pwq->work_color = work_color;
  2210. }
  2211. spin_unlock_irq(&pool->lock);
  2212. }
  2213. if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
  2214. complete(&wq->first_flusher->done);
  2215. return wait;
  2216. }
  2217. /**
  2218. * flush_workqueue - ensure that any scheduled work has run to completion.
  2219. * @wq: workqueue to flush
  2220. *
  2221. * This function sleeps until all work items which were queued on entry
  2222. * have finished execution, but it is not livelocked by new incoming ones.
  2223. */
  2224. void flush_workqueue(struct workqueue_struct *wq)
  2225. {
  2226. struct wq_flusher this_flusher = {
  2227. .list = LIST_HEAD_INIT(this_flusher.list),
  2228. .flush_color = -1,
  2229. .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
  2230. };
  2231. int next_color;
  2232. lock_map_acquire(&wq->lockdep_map);
  2233. lock_map_release(&wq->lockdep_map);
  2234. mutex_lock(&wq->mutex);
  2235. /*
  2236. * Start-to-wait phase
  2237. */
  2238. next_color = work_next_color(wq->work_color);
  2239. if (next_color != wq->flush_color) {
  2240. /*
  2241. * Color space is not full. The current work_color
  2242. * becomes our flush_color and work_color is advanced
  2243. * by one.
  2244. */
  2245. WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
  2246. this_flusher.flush_color = wq->work_color;
  2247. wq->work_color = next_color;
  2248. if (!wq->first_flusher) {
  2249. /* no flush in progress, become the first flusher */
  2250. WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
  2251. wq->first_flusher = &this_flusher;
  2252. if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
  2253. wq->work_color)) {
  2254. /* nothing to flush, done */
  2255. wq->flush_color = next_color;
  2256. wq->first_flusher = NULL;
  2257. goto out_unlock;
  2258. }
  2259. } else {
  2260. /* wait in queue */
  2261. WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
  2262. list_add_tail(&this_flusher.list, &wq->flusher_queue);
  2263. flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
  2264. }
  2265. } else {
  2266. /*
  2267. * Oops, color space is full, wait on overflow queue.
  2268. * The next flush completion will assign us
  2269. * flush_color and transfer to flusher_queue.
  2270. */
  2271. list_add_tail(&this_flusher.list, &wq->flusher_overflow);
  2272. }
  2273. mutex_unlock(&wq->mutex);
  2274. wait_for_completion(&this_flusher.done);
  2275. /*
  2276. * Wake-up-and-cascade phase
  2277. *
  2278. * First flushers are responsible for cascading flushes and
  2279. * handling overflow. Non-first flushers can simply return.
  2280. */
  2281. if (wq->first_flusher != &this_flusher)
  2282. return;
  2283. mutex_lock(&wq->mutex);
  2284. /* we might have raced, check again with mutex held */
  2285. if (wq->first_flusher != &this_flusher)
  2286. goto out_unlock;
  2287. wq->first_flusher = NULL;
  2288. WARN_ON_ONCE(!list_empty(&this_flusher.list));
  2289. WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
  2290. while (true) {
  2291. struct wq_flusher *next, *tmp;
  2292. /* complete all the flushers sharing the current flush color */
  2293. list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
  2294. if (next->flush_color != wq->flush_color)
  2295. break;
  2296. list_del_init(&next->list);
  2297. complete(&next->done);
  2298. }
  2299. WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
  2300. wq->flush_color != work_next_color(wq->work_color));
  2301. /* this flush_color is finished, advance by one */
  2302. wq->flush_color = work_next_color(wq->flush_color);
  2303. /* one color has been freed, handle overflow queue */
  2304. if (!list_empty(&wq->flusher_overflow)) {
  2305. /*
  2306. * Assign the same color to all overflowed
  2307. * flushers, advance work_color and append to
  2308. * flusher_queue. This is the start-to-wait
  2309. * phase for these overflowed flushers.
  2310. */
  2311. list_for_each_entry(tmp, &wq->flusher_overflow, list)
  2312. tmp->flush_color = wq->work_color;
  2313. wq->work_color = work_next_color(wq->work_color);
  2314. list_splice_tail_init(&wq->flusher_overflow,
  2315. &wq->flusher_queue);
  2316. flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
  2317. }
  2318. if (list_empty(&wq->flusher_queue)) {
  2319. WARN_ON_ONCE(wq->flush_color != wq->work_color);
  2320. break;
  2321. }
  2322. /*
  2323. * Need to flush more colors. Make the next flusher
  2324. * the new first flusher and arm pwqs.
  2325. */
  2326. WARN_ON_ONCE(wq->flush_color == wq->work_color);
  2327. WARN_ON_ONCE(wq->flush_color != next->flush_color);
  2328. list_del_init(&next->list);
  2329. wq->first_flusher = next;
  2330. if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
  2331. break;
  2332. /*
  2333. * Meh... this color is already done, clear first
  2334. * flusher and repeat cascading.
  2335. */
  2336. wq->first_flusher = NULL;
  2337. }
  2338. out_unlock:
  2339. mutex_unlock(&wq->mutex);
  2340. }
  2341. EXPORT_SYMBOL_GPL(flush_workqueue);
  2342. /**
  2343. * drain_workqueue - drain a workqueue
  2344. * @wq: workqueue to drain
  2345. *
  2346. * Wait until the workqueue becomes empty. While draining is in progress,
  2347. * only chain queueing is allowed. IOW, only currently pending or running
  2348. * work items on @wq can queue further work items on it. @wq is flushed
  2349. * repeatedly until it becomes empty. The number of flushing is detemined
  2350. * by the depth of chaining and should be relatively short. Whine if it
  2351. * takes too long.
  2352. */
  2353. void drain_workqueue(struct workqueue_struct *wq)
  2354. {
  2355. unsigned int flush_cnt = 0;
  2356. struct pool_workqueue *pwq;
  2357. /*
  2358. * __queue_work() needs to test whether there are drainers, is much
  2359. * hotter than drain_workqueue() and already looks at @wq->flags.
  2360. * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
  2361. */
  2362. mutex_lock(&wq->mutex);
  2363. if (!wq->nr_drainers++)
  2364. wq->flags |= __WQ_DRAINING;
  2365. mutex_unlock(&wq->mutex);
  2366. reflush:
  2367. flush_workqueue(wq);
  2368. mutex_lock(&wq->mutex);
  2369. for_each_pwq(pwq, wq) {
  2370. bool drained;
  2371. spin_lock_irq(&pwq->pool->lock);
  2372. drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
  2373. spin_unlock_irq(&pwq->pool->lock);
  2374. if (drained)
  2375. continue;
  2376. if (++flush_cnt == 10 ||
  2377. (flush_cnt % 100 == 0 && flush_cnt <= 1000))
  2378. pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
  2379. wq->name, flush_cnt);
  2380. mutex_unlock(&wq->mutex);
  2381. goto reflush;
  2382. }
  2383. if (!--wq->nr_drainers)
  2384. wq->flags &= ~__WQ_DRAINING;
  2385. mutex_unlock(&wq->mutex);
  2386. }
  2387. EXPORT_SYMBOL_GPL(drain_workqueue);
  2388. static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
  2389. {
  2390. struct worker *worker = NULL;
  2391. struct worker_pool *pool;
  2392. struct pool_workqueue *pwq;
  2393. might_sleep();
  2394. local_irq_disable();
  2395. pool = get_work_pool(work);
  2396. if (!pool) {
  2397. local_irq_enable();
  2398. return false;
  2399. }
  2400. spin_lock(&pool->lock);
  2401. /* see the comment in try_to_grab_pending() with the same code */
  2402. pwq = get_work_pwq(work);
  2403. if (pwq) {
  2404. if (unlikely(pwq->pool != pool))
  2405. goto already_gone;
  2406. } else {
  2407. worker = find_worker_executing_work(pool, work);
  2408. if (!worker)
  2409. goto already_gone;
  2410. pwq = worker->current_pwq;
  2411. }
  2412. insert_wq_barrier(pwq, barr, work, worker);
  2413. spin_unlock_irq(&pool->lock);
  2414. /*
  2415. * If @max_active is 1 or rescuer is in use, flushing another work
  2416. * item on the same workqueue may lead to deadlock. Make sure the
  2417. * flusher is not running on the same workqueue by verifying write
  2418. * access.
  2419. */
  2420. if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)
  2421. lock_map_acquire(&pwq->wq->lockdep_map);
  2422. else
  2423. lock_map_acquire_read(&pwq->wq->lockdep_map);
  2424. lock_map_release(&pwq->wq->lockdep_map);
  2425. return true;
  2426. already_gone:
  2427. spin_unlock_irq(&pool->lock);
  2428. return false;
  2429. }
  2430. /**
  2431. * flush_work - wait for a work to finish executing the last queueing instance
  2432. * @work: the work to flush
  2433. *
  2434. * Wait until @work has finished execution. @work is guaranteed to be idle
  2435. * on return if it hasn't been requeued since flush started.
  2436. *
  2437. * RETURNS:
  2438. * %true if flush_work() waited for the work to finish execution,
  2439. * %false if it was already idle.
  2440. */
  2441. bool flush_work(struct work_struct *work)
  2442. {
  2443. struct wq_barrier barr;
  2444. lock_map_acquire(&work->lockdep_map);
  2445. lock_map_release(&work->lockdep_map);
  2446. if (start_flush_work(work, &barr)) {
  2447. wait_for_completion(&barr.done);
  2448. destroy_work_on_stack(&barr.work);
  2449. return true;
  2450. } else {
  2451. return false;
  2452. }
  2453. }
  2454. EXPORT_SYMBOL_GPL(flush_work);
  2455. static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
  2456. {
  2457. unsigned long flags;
  2458. int ret;
  2459. do {
  2460. ret = try_to_grab_pending(work, is_dwork, &flags);
  2461. /*
  2462. * If someone else is canceling, wait for the same event it
  2463. * would be waiting for before retrying.
  2464. */
  2465. if (unlikely(ret == -ENOENT))
  2466. flush_work(work);
  2467. } while (unlikely(ret < 0));
  2468. /* tell other tasks trying to grab @work to back off */
  2469. mark_work_canceling(work);
  2470. local_irq_restore(flags);
  2471. flush_work(work);
  2472. clear_work_data(work);
  2473. return ret;
  2474. }
  2475. /**
  2476. * cancel_work_sync - cancel a work and wait for it to finish
  2477. * @work: the work to cancel
  2478. *
  2479. * Cancel @work and wait for its execution to finish. This function
  2480. * can be used even if the work re-queues itself or migrates to
  2481. * another workqueue. On return from this function, @work is
  2482. * guaranteed to be not pending or executing on any CPU.
  2483. *
  2484. * cancel_work_sync(&delayed_work->work) must not be used for
  2485. * delayed_work's. Use cancel_delayed_work_sync() instead.
  2486. *
  2487. * The caller must ensure that the workqueue on which @work was last
  2488. * queued can't be destroyed before this function returns.
  2489. *
  2490. * RETURNS:
  2491. * %true if @work was pending, %false otherwise.
  2492. */
  2493. bool cancel_work_sync(struct work_struct *work)
  2494. {
  2495. return __cancel_work_timer(work, false);
  2496. }
  2497. EXPORT_SYMBOL_GPL(cancel_work_sync);
  2498. /**
  2499. * flush_delayed_work - wait for a dwork to finish executing the last queueing
  2500. * @dwork: the delayed work to flush
  2501. *
  2502. * Delayed timer is cancelled and the pending work is queued for
  2503. * immediate execution. Like flush_work(), this function only
  2504. * considers the last queueing instance of @dwork.
  2505. *
  2506. * RETURNS:
  2507. * %true if flush_work() waited for the work to finish execution,
  2508. * %false if it was already idle.
  2509. */
  2510. bool flush_delayed_work(struct delayed_work *dwork)
  2511. {
  2512. local_irq_disable();
  2513. if (del_timer_sync(&dwork->timer))
  2514. __queue_work(dwork->cpu, dwork->wq, &dwork->work);
  2515. local_irq_enable();
  2516. return flush_work(&dwork->work);
  2517. }
  2518. EXPORT_SYMBOL(flush_delayed_work);
  2519. /**
  2520. * cancel_delayed_work - cancel a delayed work
  2521. * @dwork: delayed_work to cancel
  2522. *
  2523. * Kill off a pending delayed_work. Returns %true if @dwork was pending
  2524. * and canceled; %false if wasn't pending. Note that the work callback
  2525. * function may still be running on return, unless it returns %true and the
  2526. * work doesn't re-arm itself. Explicitly flush or use
  2527. * cancel_delayed_work_sync() to wait on it.
  2528. *
  2529. * This function is safe to call from any context including IRQ handler.
  2530. */
  2531. bool cancel_delayed_work(struct delayed_work *dwork)
  2532. {
  2533. unsigned long flags;
  2534. int ret;
  2535. do {
  2536. ret = try_to_grab_pending(&dwork->work, true, &flags);
  2537. } while (unlikely(ret == -EAGAIN));
  2538. if (unlikely(ret < 0))
  2539. return false;
  2540. set_work_pool_and_clear_pending(&dwork->work,
  2541. get_work_pool_id(&dwork->work));
  2542. local_irq_restore(flags);
  2543. return ret;
  2544. }
  2545. EXPORT_SYMBOL(cancel_delayed_work);
  2546. /**
  2547. * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
  2548. * @dwork: the delayed work cancel
  2549. *
  2550. * This is cancel_work_sync() for delayed works.
  2551. *
  2552. * RETURNS:
  2553. * %true if @dwork was pending, %false otherwise.
  2554. */
  2555. bool cancel_delayed_work_sync(struct delayed_work *dwork)
  2556. {
  2557. return __cancel_work_timer(&dwork->work, true);
  2558. }
  2559. EXPORT_SYMBOL(cancel_delayed_work_sync);
  2560. /**
  2561. * schedule_on_each_cpu - execute a function synchronously on each online CPU
  2562. * @func: the function to call
  2563. *
  2564. * schedule_on_each_cpu() executes @func on each online CPU using the
  2565. * system workqueue and blocks until all CPUs have completed.
  2566. * schedule_on_each_cpu() is very slow.
  2567. *
  2568. * RETURNS:
  2569. * 0 on success, -errno on failure.
  2570. */
  2571. int schedule_on_each_cpu(work_func_t func)
  2572. {
  2573. int cpu;
  2574. struct work_struct __percpu *works;
  2575. works = alloc_percpu(struct work_struct);
  2576. if (!works)
  2577. return -ENOMEM;
  2578. get_online_cpus();
  2579. for_each_online_cpu(cpu) {
  2580. struct work_struct *work = per_cpu_ptr(works, cpu);
  2581. INIT_WORK(work, func);
  2582. schedule_work_on(cpu, work);
  2583. }
  2584. for_each_online_cpu(cpu)
  2585. flush_work(per_cpu_ptr(works, cpu));
  2586. put_online_cpus();
  2587. free_percpu(works);
  2588. return 0;
  2589. }
  2590. /**
  2591. * flush_scheduled_work - ensure that any scheduled work has run to completion.
  2592. *
  2593. * Forces execution of the kernel-global workqueue and blocks until its
  2594. * completion.
  2595. *
  2596. * Think twice before calling this function! It's very easy to get into
  2597. * trouble if you don't take great care. Either of the following situations
  2598. * will lead to deadlock:
  2599. *
  2600. * One of the work items currently on the workqueue needs to acquire
  2601. * a lock held by your code or its caller.
  2602. *
  2603. * Your code is running in the context of a work routine.
  2604. *
  2605. * They will be detected by lockdep when they occur, but the first might not
  2606. * occur very often. It depends on what work items are on the workqueue and
  2607. * what locks they need, which you have no control over.
  2608. *
  2609. * In most situations flushing the entire workqueue is overkill; you merely
  2610. * need to know that a particular work item isn't queued and isn't running.
  2611. * In such cases you should use cancel_delayed_work_sync() or
  2612. * cancel_work_sync() instead.
  2613. */
  2614. void flush_scheduled_work(void)
  2615. {
  2616. flush_workqueue(system_wq);
  2617. }
  2618. EXPORT_SYMBOL(flush_scheduled_work);
  2619. /**
  2620. * execute_in_process_context - reliably execute the routine with user context
  2621. * @fn: the function to execute
  2622. * @ew: guaranteed storage for the execute work structure (must
  2623. * be available when the work executes)
  2624. *
  2625. * Executes the function immediately if process context is available,
  2626. * otherwise schedules the function for delayed execution.
  2627. *
  2628. * Returns: 0 - function was executed
  2629. * 1 - function was scheduled for execution
  2630. */
  2631. int execute_in_process_context(work_func_t fn, struct execute_work *ew)
  2632. {
  2633. if (!in_interrupt()) {
  2634. fn(&ew->work);
  2635. return 0;
  2636. }
  2637. INIT_WORK(&ew->work, fn);
  2638. schedule_work(&ew->work);
  2639. return 1;
  2640. }
  2641. EXPORT_SYMBOL_GPL(execute_in_process_context);
  2642. #ifdef CONFIG_SYSFS
  2643. /*
  2644. * Workqueues with WQ_SYSFS flag set is visible to userland via
  2645. * /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
  2646. * following attributes.
  2647. *
  2648. * per_cpu RO bool : whether the workqueue is per-cpu or unbound
  2649. * max_active RW int : maximum number of in-flight work items
  2650. *
  2651. * Unbound workqueues have the following extra attributes.
  2652. *
  2653. * id RO int : the associated pool ID
  2654. * nice RW int : nice value of the workers
  2655. * cpumask RW mask : bitmask of allowed CPUs for the workers
  2656. */
  2657. struct wq_device {
  2658. struct workqueue_struct *wq;
  2659. struct device dev;
  2660. };
  2661. static struct workqueue_struct *dev_to_wq(struct device *dev)
  2662. {
  2663. struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
  2664. return wq_dev->wq;
  2665. }
  2666. static ssize_t wq_per_cpu_show(struct device *dev,
  2667. struct device_attribute *attr, char *buf)
  2668. {
  2669. struct workqueue_struct *wq = dev_to_wq(dev);
  2670. return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
  2671. }
  2672. static ssize_t wq_max_active_show(struct device *dev,
  2673. struct device_attribute *attr, char *buf)
  2674. {
  2675. struct workqueue_struct *wq = dev_to_wq(dev);
  2676. return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
  2677. }
  2678. static ssize_t wq_max_active_store(struct device *dev,
  2679. struct device_attribute *attr,
  2680. const char *buf, size_t count)
  2681. {
  2682. struct workqueue_struct *wq = dev_to_wq(dev);
  2683. int val;
  2684. if (sscanf(buf, "%d", &val) != 1 || val <= 0)
  2685. return -EINVAL;
  2686. workqueue_set_max_active(wq, val);
  2687. return count;
  2688. }
  2689. static struct device_attribute wq_sysfs_attrs[] = {
  2690. __ATTR(per_cpu, 0444, wq_per_cpu_show, NULL),
  2691. __ATTR(max_active, 0644, wq_max_active_show, wq_max_active_store),
  2692. __ATTR_NULL,
  2693. };
  2694. static ssize_t wq_pool_id_show(struct device *dev,
  2695. struct device_attribute *attr, char *buf)
  2696. {
  2697. struct workqueue_struct *wq = dev_to_wq(dev);
  2698. struct worker_pool *pool;
  2699. int written;
  2700. rcu_read_lock_sched();
  2701. pool = first_pwq(wq)->pool;
  2702. written = scnprintf(buf, PAGE_SIZE, "%d\n", pool->id);
  2703. rcu_read_unlock_sched();
  2704. return written;
  2705. }
  2706. static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
  2707. char *buf)
  2708. {
  2709. struct workqueue_struct *wq = dev_to_wq(dev);
  2710. int written;
  2711. mutex_lock(&wq->mutex);
  2712. written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
  2713. mutex_unlock(&wq->mutex);
  2714. return written;
  2715. }
  2716. /* prepare workqueue_attrs for sysfs store operations */
  2717. static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
  2718. {
  2719. struct workqueue_attrs *attrs;
  2720. attrs = alloc_workqueue_attrs(GFP_KERNEL);
  2721. if (!attrs)
  2722. return NULL;
  2723. mutex_lock(&wq->mutex);
  2724. copy_workqueue_attrs(attrs, wq->unbound_attrs);
  2725. mutex_unlock(&wq->mutex);
  2726. return attrs;
  2727. }
  2728. static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
  2729. const char *buf, size_t count)
  2730. {
  2731. struct workqueue_struct *wq = dev_to_wq(dev);
  2732. struct workqueue_attrs *attrs;
  2733. int ret;
  2734. attrs = wq_sysfs_prep_attrs(wq);
  2735. if (!attrs)
  2736. return -ENOMEM;
  2737. if (sscanf(buf, "%d", &attrs->nice) == 1 &&
  2738. attrs->nice >= -20 && attrs->nice <= 19)
  2739. ret = apply_workqueue_attrs(wq, attrs);
  2740. else
  2741. ret = -EINVAL;
  2742. free_workqueue_attrs(attrs);
  2743. return ret ?: count;
  2744. }
  2745. static ssize_t wq_cpumask_show(struct device *dev,
  2746. struct device_attribute *attr, char *buf)
  2747. {
  2748. struct workqueue_struct *wq = dev_to_wq(dev);
  2749. int written;
  2750. mutex_lock(&wq->mutex);
  2751. written = cpumask_scnprintf(buf, PAGE_SIZE, wq->unbound_attrs->cpumask);
  2752. mutex_unlock(&wq->mutex);
  2753. written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
  2754. return written;
  2755. }
  2756. static ssize_t wq_cpumask_store(struct device *dev,
  2757. struct device_attribute *attr,
  2758. const char *buf, size_t count)
  2759. {
  2760. struct workqueue_struct *wq = dev_to_wq(dev);
  2761. struct workqueue_attrs *attrs;
  2762. int ret;
  2763. attrs = wq_sysfs_prep_attrs(wq);
  2764. if (!attrs)
  2765. return -ENOMEM;
  2766. ret = cpumask_parse(buf, attrs->cpumask);
  2767. if (!ret)
  2768. ret = apply_workqueue_attrs(wq, attrs);
  2769. free_workqueue_attrs(attrs);
  2770. return ret ?: count;
  2771. }
  2772. static struct device_attribute wq_sysfs_unbound_attrs[] = {
  2773. __ATTR(pool_id, 0444, wq_pool_id_show, NULL),
  2774. __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
  2775. __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
  2776. __ATTR_NULL,
  2777. };
  2778. static struct bus_type wq_subsys = {
  2779. .name = "workqueue",
  2780. .dev_attrs = wq_sysfs_attrs,
  2781. };
  2782. static int __init wq_sysfs_init(void)
  2783. {
  2784. return subsys_virtual_register(&wq_subsys, NULL);
  2785. }
  2786. core_initcall(wq_sysfs_init);
  2787. static void wq_device_release(struct device *dev)
  2788. {
  2789. struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
  2790. kfree(wq_dev);
  2791. }
  2792. /**
  2793. * workqueue_sysfs_register - make a workqueue visible in sysfs
  2794. * @wq: the workqueue to register
  2795. *
  2796. * Expose @wq in sysfs under /sys/bus/workqueue/devices.
  2797. * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
  2798. * which is the preferred method.
  2799. *
  2800. * Workqueue user should use this function directly iff it wants to apply
  2801. * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
  2802. * apply_workqueue_attrs() may race against userland updating the
  2803. * attributes.
  2804. *
  2805. * Returns 0 on success, -errno on failure.
  2806. */
  2807. int workqueue_sysfs_register(struct workqueue_struct *wq)
  2808. {
  2809. struct wq_device *wq_dev;
  2810. int ret;
  2811. /*
  2812. * Adjusting max_active or creating new pwqs by applyting
  2813. * attributes breaks ordering guarantee. Disallow exposing ordered
  2814. * workqueues.
  2815. */
  2816. if (WARN_ON(wq->flags & __WQ_ORDERED))
  2817. return -EINVAL;
  2818. wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
  2819. if (!wq_dev)
  2820. return -ENOMEM;
  2821. wq_dev->wq = wq;
  2822. wq_dev->dev.bus = &wq_subsys;
  2823. wq_dev->dev.init_name = wq->name;
  2824. wq_dev->dev.release = wq_device_release;
  2825. /*
  2826. * unbound_attrs are created separately. Suppress uevent until
  2827. * everything is ready.
  2828. */
  2829. dev_set_uevent_suppress(&wq_dev->dev, true);
  2830. ret = device_register(&wq_dev->dev);
  2831. if (ret) {
  2832. kfree(wq_dev);
  2833. wq->wq_dev = NULL;
  2834. return ret;
  2835. }
  2836. if (wq->flags & WQ_UNBOUND) {
  2837. struct device_attribute *attr;
  2838. for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
  2839. ret = device_create_file(&wq_dev->dev, attr);
  2840. if (ret) {
  2841. device_unregister(&wq_dev->dev);
  2842. wq->wq_dev = NULL;
  2843. return ret;
  2844. }
  2845. }
  2846. }
  2847. kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
  2848. return 0;
  2849. }
  2850. /**
  2851. * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
  2852. * @wq: the workqueue to unregister
  2853. *
  2854. * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
  2855. */
  2856. static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
  2857. {
  2858. struct wq_device *wq_dev = wq->wq_dev;
  2859. if (!wq->wq_dev)
  2860. return;
  2861. wq->wq_dev = NULL;
  2862. device_unregister(&wq_dev->dev);
  2863. }
  2864. #else /* CONFIG_SYSFS */
  2865. static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
  2866. #endif /* CONFIG_SYSFS */
  2867. /**
  2868. * free_workqueue_attrs - free a workqueue_attrs
  2869. * @attrs: workqueue_attrs to free
  2870. *
  2871. * Undo alloc_workqueue_attrs().
  2872. */
  2873. void free_workqueue_attrs(struct workqueue_attrs *attrs)
  2874. {
  2875. if (attrs) {
  2876. free_cpumask_var(attrs->cpumask);
  2877. kfree(attrs);
  2878. }
  2879. }
  2880. /**
  2881. * alloc_workqueue_attrs - allocate a workqueue_attrs
  2882. * @gfp_mask: allocation mask to use
  2883. *
  2884. * Allocate a new workqueue_attrs, initialize with default settings and
  2885. * return it. Returns NULL on failure.
  2886. */
  2887. struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
  2888. {
  2889. struct workqueue_attrs *attrs;
  2890. attrs = kzalloc(sizeof(*attrs), gfp_mask);
  2891. if (!attrs)
  2892. goto fail;
  2893. if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
  2894. goto fail;
  2895. cpumask_copy(attrs->cpumask, cpu_possible_mask);
  2896. return attrs;
  2897. fail:
  2898. free_workqueue_attrs(attrs);
  2899. return NULL;
  2900. }
  2901. static void copy_workqueue_attrs(struct workqueue_attrs *to,
  2902. const struct workqueue_attrs *from)
  2903. {
  2904. to->nice = from->nice;
  2905. cpumask_copy(to->cpumask, from->cpumask);
  2906. }
  2907. /* hash value of the content of @attr */
  2908. static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
  2909. {
  2910. u32 hash = 0;
  2911. hash = jhash_1word(attrs->nice, hash);
  2912. hash = jhash(cpumask_bits(attrs->cpumask),
  2913. BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
  2914. return hash;
  2915. }
  2916. /* content equality test */
  2917. static bool wqattrs_equal(const struct workqueue_attrs *a,
  2918. const struct workqueue_attrs *b)
  2919. {
  2920. if (a->nice != b->nice)
  2921. return false;
  2922. if (!cpumask_equal(a->cpumask, b->cpumask))
  2923. return false;
  2924. return true;
  2925. }
  2926. /**
  2927. * init_worker_pool - initialize a newly zalloc'd worker_pool
  2928. * @pool: worker_pool to initialize
  2929. *
  2930. * Initiailize a newly zalloc'd @pool. It also allocates @pool->attrs.
  2931. * Returns 0 on success, -errno on failure. Even on failure, all fields
  2932. * inside @pool proper are initialized and put_unbound_pool() can be called
  2933. * on @pool safely to release it.
  2934. */
  2935. static int init_worker_pool(struct worker_pool *pool)
  2936. {
  2937. spin_lock_init(&pool->lock);
  2938. pool->id = -1;
  2939. pool->cpu = -1;
  2940. pool->node = NUMA_NO_NODE;
  2941. pool->flags |= POOL_DISASSOCIATED;
  2942. INIT_LIST_HEAD(&pool->worklist);
  2943. INIT_LIST_HEAD(&pool->idle_list);
  2944. hash_init(pool->busy_hash);
  2945. init_timer_deferrable(&pool->idle_timer);
  2946. pool->idle_timer.function = idle_worker_timeout;
  2947. pool->idle_timer.data = (unsigned long)pool;
  2948. setup_timer(&pool->mayday_timer, pool_mayday_timeout,
  2949. (unsigned long)pool);
  2950. mutex_init(&pool->manager_arb);
  2951. mutex_init(&pool->manager_mutex);
  2952. idr_init(&pool->worker_idr);
  2953. INIT_HLIST_NODE(&pool->hash_node);
  2954. pool->refcnt = 1;
  2955. /* shouldn't fail above this point */
  2956. pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
  2957. if (!pool->attrs)
  2958. return -ENOMEM;
  2959. return 0;
  2960. }
  2961. static void rcu_free_pool(struct rcu_head *rcu)
  2962. {
  2963. struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
  2964. idr_destroy(&pool->worker_idr);
  2965. free_workqueue_attrs(pool->attrs);
  2966. kfree(pool);
  2967. }
  2968. /**
  2969. * put_unbound_pool - put a worker_pool
  2970. * @pool: worker_pool to put
  2971. *
  2972. * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
  2973. * safe manner. get_unbound_pool() calls this function on its failure path
  2974. * and this function should be able to release pools which went through,
  2975. * successfully or not, init_worker_pool().
  2976. *
  2977. * Should be called with wq_pool_mutex held.
  2978. */
  2979. static void put_unbound_pool(struct worker_pool *pool)
  2980. {
  2981. struct worker *worker;
  2982. lockdep_assert_held(&wq_pool_mutex);
  2983. if (--pool->refcnt)
  2984. return;
  2985. /* sanity checks */
  2986. if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) ||
  2987. WARN_ON(!list_empty(&pool->worklist)))
  2988. return;
  2989. /* release id and unhash */
  2990. if (pool->id >= 0)
  2991. idr_remove(&worker_pool_idr, pool->id);
  2992. hash_del(&pool->hash_node);
  2993. /*
  2994. * Become the manager and destroy all workers. Grabbing
  2995. * manager_arb prevents @pool's workers from blocking on
  2996. * manager_mutex.
  2997. */
  2998. mutex_lock(&pool->manager_arb);
  2999. mutex_lock(&pool->manager_mutex);
  3000. spin_lock_irq(&pool->lock);
  3001. while ((worker = first_worker(pool)))
  3002. destroy_worker(worker);
  3003. WARN_ON(pool->nr_workers || pool->nr_idle);
  3004. spin_unlock_irq(&pool->lock);
  3005. mutex_unlock(&pool->manager_mutex);
  3006. mutex_unlock(&pool->manager_arb);
  3007. /* shut down the timers */
  3008. del_timer_sync(&pool->idle_timer);
  3009. del_timer_sync(&pool->mayday_timer);
  3010. /* sched-RCU protected to allow dereferences from get_work_pool() */
  3011. call_rcu_sched(&pool->rcu, rcu_free_pool);
  3012. }
  3013. /**
  3014. * get_unbound_pool - get a worker_pool with the specified attributes
  3015. * @attrs: the attributes of the worker_pool to get
  3016. *
  3017. * Obtain a worker_pool which has the same attributes as @attrs, bump the
  3018. * reference count and return it. If there already is a matching
  3019. * worker_pool, it will be used; otherwise, this function attempts to
  3020. * create a new one. On failure, returns NULL.
  3021. *
  3022. * Should be called with wq_pool_mutex held.
  3023. */
  3024. static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
  3025. {
  3026. u32 hash = wqattrs_hash(attrs);
  3027. struct worker_pool *pool;
  3028. int node;
  3029. lockdep_assert_held(&wq_pool_mutex);
  3030. /* do we already have a matching pool? */
  3031. hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
  3032. if (wqattrs_equal(pool->attrs, attrs)) {
  3033. pool->refcnt++;
  3034. goto out_unlock;
  3035. }
  3036. }
  3037. /* nope, create a new one */
  3038. pool = kzalloc(sizeof(*pool), GFP_KERNEL);
  3039. if (!pool || init_worker_pool(pool) < 0)
  3040. goto fail;
  3041. if (workqueue_freezing)
  3042. pool->flags |= POOL_FREEZING;
  3043. lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
  3044. copy_workqueue_attrs(pool->attrs, attrs);
  3045. /* if cpumask is contained inside a NUMA node, we belong to that node */
  3046. if (wq_numa_enabled) {
  3047. for_each_node(node) {
  3048. if (cpumask_subset(pool->attrs->cpumask,
  3049. wq_numa_possible_cpumask[node])) {
  3050. pool->node = node;
  3051. break;
  3052. }
  3053. }
  3054. }
  3055. if (worker_pool_assign_id(pool) < 0)
  3056. goto fail;
  3057. /* create and start the initial worker */
  3058. if (create_and_start_worker(pool) < 0)
  3059. goto fail;
  3060. /* install */
  3061. hash_add(unbound_pool_hash, &pool->hash_node, hash);
  3062. out_unlock:
  3063. return pool;
  3064. fail:
  3065. if (pool)
  3066. put_unbound_pool(pool);
  3067. return NULL;
  3068. }
  3069. static void rcu_free_pwq(struct rcu_head *rcu)
  3070. {
  3071. kmem_cache_free(pwq_cache,
  3072. container_of(rcu, struct pool_workqueue, rcu));
  3073. }
  3074. /*
  3075. * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
  3076. * and needs to be destroyed.
  3077. */
  3078. static void pwq_unbound_release_workfn(struct work_struct *work)
  3079. {
  3080. struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
  3081. unbound_release_work);
  3082. struct workqueue_struct *wq = pwq->wq;
  3083. struct worker_pool *pool = pwq->pool;
  3084. bool is_last;
  3085. if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
  3086. return;
  3087. /*
  3088. * Unlink @pwq. Synchronization against wq->mutex isn't strictly
  3089. * necessary on release but do it anyway. It's easier to verify
  3090. * and consistent with the linking path.
  3091. */
  3092. mutex_lock(&wq->mutex);
  3093. list_del_rcu(&pwq->pwqs_node);
  3094. is_last = list_empty(&wq->pwqs);
  3095. mutex_unlock(&wq->mutex);
  3096. mutex_lock(&wq_pool_mutex);
  3097. put_unbound_pool(pool);
  3098. mutex_unlock(&wq_pool_mutex);
  3099. call_rcu_sched(&pwq->rcu, rcu_free_pwq);
  3100. /*
  3101. * If we're the last pwq going away, @wq is already dead and no one
  3102. * is gonna access it anymore. Free it.
  3103. */
  3104. if (is_last) {
  3105. free_workqueue_attrs(wq->unbound_attrs);
  3106. kfree(wq);
  3107. }
  3108. }
  3109. /**
  3110. * pwq_adjust_max_active - update a pwq's max_active to the current setting
  3111. * @pwq: target pool_workqueue
  3112. *
  3113. * If @pwq isn't freezing, set @pwq->max_active to the associated
  3114. * workqueue's saved_max_active and activate delayed work items
  3115. * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
  3116. */
  3117. static void pwq_adjust_max_active(struct pool_workqueue *pwq)
  3118. {
  3119. struct workqueue_struct *wq = pwq->wq;
  3120. bool freezable = wq->flags & WQ_FREEZABLE;
  3121. /* for @wq->saved_max_active */
  3122. lockdep_assert_held(&wq->mutex);
  3123. /* fast exit for non-freezable wqs */
  3124. if (!freezable && pwq->max_active == wq->saved_max_active)
  3125. return;
  3126. spin_lock_irq(&pwq->pool->lock);
  3127. if (!freezable || !(pwq->pool->flags & POOL_FREEZING)) {
  3128. pwq->max_active = wq->saved_max_active;
  3129. while (!list_empty(&pwq->delayed_works) &&
  3130. pwq->nr_active < pwq->max_active)
  3131. pwq_activate_first_delayed(pwq);
  3132. /*
  3133. * Need to kick a worker after thawed or an unbound wq's
  3134. * max_active is bumped. It's a slow path. Do it always.
  3135. */
  3136. wake_up_worker(pwq->pool);
  3137. } else {
  3138. pwq->max_active = 0;
  3139. }
  3140. spin_unlock_irq(&pwq->pool->lock);
  3141. }
  3142. /* initialize newly alloced @pwq which is associated with @wq and @pool */
  3143. static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
  3144. struct worker_pool *pool)
  3145. {
  3146. BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
  3147. memset(pwq, 0, sizeof(*pwq));
  3148. pwq->pool = pool;
  3149. pwq->wq = wq;
  3150. pwq->flush_color = -1;
  3151. pwq->refcnt = 1;
  3152. INIT_LIST_HEAD(&pwq->delayed_works);
  3153. INIT_LIST_HEAD(&pwq->pwqs_node);
  3154. INIT_LIST_HEAD(&pwq->mayday_node);
  3155. INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
  3156. }
  3157. /* sync @pwq with the current state of its associated wq and link it */
  3158. static void link_pwq(struct pool_workqueue *pwq)
  3159. {
  3160. struct workqueue_struct *wq = pwq->wq;
  3161. lockdep_assert_held(&wq->mutex);
  3162. /* may be called multiple times, ignore if already linked */
  3163. if (!list_empty(&pwq->pwqs_node))
  3164. return;
  3165. /*
  3166. * Set the matching work_color. This is synchronized with
  3167. * wq->mutex to avoid confusing flush_workqueue().
  3168. */
  3169. pwq->work_color = wq->work_color;
  3170. /* sync max_active to the current setting */
  3171. pwq_adjust_max_active(pwq);
  3172. /* link in @pwq */
  3173. list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
  3174. }
  3175. /* obtain a pool matching @attr and create a pwq associating the pool and @wq */
  3176. static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
  3177. const struct workqueue_attrs *attrs)
  3178. {
  3179. struct worker_pool *pool;
  3180. struct pool_workqueue *pwq;
  3181. lockdep_assert_held(&wq_pool_mutex);
  3182. pool = get_unbound_pool(attrs);
  3183. if (!pool)
  3184. return NULL;
  3185. pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
  3186. if (!pwq) {
  3187. put_unbound_pool(pool);
  3188. return NULL;
  3189. }
  3190. init_pwq(pwq, wq, pool);
  3191. return pwq;
  3192. }
  3193. /* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
  3194. static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
  3195. int node,
  3196. struct pool_workqueue *pwq)
  3197. {
  3198. struct pool_workqueue *old_pwq;
  3199. lockdep_assert_held(&wq->mutex);
  3200. /* link_pwq() can handle duplicate calls */
  3201. link_pwq(pwq);
  3202. old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
  3203. rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
  3204. return old_pwq;
  3205. }
  3206. /**
  3207. * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
  3208. * @wq: the target workqueue
  3209. * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
  3210. *
  3211. * Apply @attrs to an unbound workqueue @wq. If @attrs doesn't match the
  3212. * current attributes, a new pwq is created and made the first pwq which
  3213. * will serve all new work items. Older pwqs are released as in-flight
  3214. * work items finish. Note that a work item which repeatedly requeues
  3215. * itself back-to-back will stay on its current pwq.
  3216. *
  3217. * Performs GFP_KERNEL allocations. Returns 0 on success and -errno on
  3218. * failure.
  3219. */
  3220. int apply_workqueue_attrs(struct workqueue_struct *wq,
  3221. const struct workqueue_attrs *attrs)
  3222. {
  3223. struct workqueue_attrs *new_attrs;
  3224. struct pool_workqueue *pwq, *last_pwq = NULL;
  3225. int node, ret;
  3226. /* only unbound workqueues can change attributes */
  3227. if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
  3228. return -EINVAL;
  3229. /* creating multiple pwqs breaks ordering guarantee */
  3230. if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
  3231. return -EINVAL;
  3232. /* make a copy of @attrs and sanitize it */
  3233. new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
  3234. if (!new_attrs)
  3235. goto enomem;
  3236. copy_workqueue_attrs(new_attrs, attrs);
  3237. cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
  3238. mutex_lock(&wq_pool_mutex);
  3239. pwq = alloc_unbound_pwq(wq, new_attrs);
  3240. mutex_unlock(&wq_pool_mutex);
  3241. if (!pwq)
  3242. goto enomem;
  3243. mutex_lock(&wq->mutex);
  3244. copy_workqueue_attrs(wq->unbound_attrs, new_attrs);
  3245. for_each_node(node)
  3246. last_pwq = numa_pwq_tbl_install(wq, node, pwq);
  3247. mutex_unlock(&wq->mutex);
  3248. put_pwq_unlocked(last_pwq);
  3249. ret = 0;
  3250. /* fall through */
  3251. out_free:
  3252. free_workqueue_attrs(new_attrs);
  3253. return ret;
  3254. enomem:
  3255. ret = -ENOMEM;
  3256. goto out_free;
  3257. }
  3258. static int alloc_and_link_pwqs(struct workqueue_struct *wq)
  3259. {
  3260. bool highpri = wq->flags & WQ_HIGHPRI;
  3261. int cpu;
  3262. if (!(wq->flags & WQ_UNBOUND)) {
  3263. wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
  3264. if (!wq->cpu_pwqs)
  3265. return -ENOMEM;
  3266. for_each_possible_cpu(cpu) {
  3267. struct pool_workqueue *pwq =
  3268. per_cpu_ptr(wq->cpu_pwqs, cpu);
  3269. struct worker_pool *cpu_pools =
  3270. per_cpu(cpu_worker_pools, cpu);
  3271. init_pwq(pwq, wq, &cpu_pools[highpri]);
  3272. mutex_lock(&wq->mutex);
  3273. link_pwq(pwq);
  3274. mutex_unlock(&wq->mutex);
  3275. }
  3276. return 0;
  3277. } else {
  3278. return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
  3279. }
  3280. }
  3281. static int wq_clamp_max_active(int max_active, unsigned int flags,
  3282. const char *name)
  3283. {
  3284. int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
  3285. if (max_active < 1 || max_active > lim)
  3286. pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
  3287. max_active, name, 1, lim);
  3288. return clamp_val(max_active, 1, lim);
  3289. }
  3290. struct workqueue_struct *__alloc_workqueue_key(const char *fmt,
  3291. unsigned int flags,
  3292. int max_active,
  3293. struct lock_class_key *key,
  3294. const char *lock_name, ...)
  3295. {
  3296. size_t tbl_size = 0;
  3297. va_list args;
  3298. struct workqueue_struct *wq;
  3299. struct pool_workqueue *pwq;
  3300. /* allocate wq and format name */
  3301. if (flags & WQ_UNBOUND)
  3302. tbl_size = wq_numa_tbl_len * sizeof(wq->numa_pwq_tbl[0]);
  3303. wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
  3304. if (!wq)
  3305. return NULL;
  3306. if (flags & WQ_UNBOUND) {
  3307. wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
  3308. if (!wq->unbound_attrs)
  3309. goto err_free_wq;
  3310. }
  3311. va_start(args, lock_name);
  3312. vsnprintf(wq->name, sizeof(wq->name), fmt, args);
  3313. va_end(args);
  3314. max_active = max_active ?: WQ_DFL_ACTIVE;
  3315. max_active = wq_clamp_max_active(max_active, flags, wq->name);
  3316. /* init wq */
  3317. wq->flags = flags;
  3318. wq->saved_max_active = max_active;
  3319. mutex_init(&wq->mutex);
  3320. atomic_set(&wq->nr_pwqs_to_flush, 0);
  3321. INIT_LIST_HEAD(&wq->pwqs);
  3322. INIT_LIST_HEAD(&wq->flusher_queue);
  3323. INIT_LIST_HEAD(&wq->flusher_overflow);
  3324. INIT_LIST_HEAD(&wq->maydays);
  3325. lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
  3326. INIT_LIST_HEAD(&wq->list);
  3327. if (alloc_and_link_pwqs(wq) < 0)
  3328. goto err_free_wq;
  3329. /*
  3330. * Workqueues which may be used during memory reclaim should
  3331. * have a rescuer to guarantee forward progress.
  3332. */
  3333. if (flags & WQ_MEM_RECLAIM) {
  3334. struct worker *rescuer;
  3335. rescuer = alloc_worker();
  3336. if (!rescuer)
  3337. goto err_destroy;
  3338. rescuer->rescue_wq = wq;
  3339. rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
  3340. wq->name);
  3341. if (IS_ERR(rescuer->task)) {
  3342. kfree(rescuer);
  3343. goto err_destroy;
  3344. }
  3345. wq->rescuer = rescuer;
  3346. rescuer->task->flags |= PF_NO_SETAFFINITY;
  3347. wake_up_process(rescuer->task);
  3348. }
  3349. if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
  3350. goto err_destroy;
  3351. /*
  3352. * wq_pool_mutex protects global freeze state and workqueues list.
  3353. * Grab it, adjust max_active and add the new @wq to workqueues
  3354. * list.
  3355. */
  3356. mutex_lock(&wq_pool_mutex);
  3357. mutex_lock(&wq->mutex);
  3358. for_each_pwq(pwq, wq)
  3359. pwq_adjust_max_active(pwq);
  3360. mutex_unlock(&wq->mutex);
  3361. list_add(&wq->list, &workqueues);
  3362. mutex_unlock(&wq_pool_mutex);
  3363. return wq;
  3364. err_free_wq:
  3365. free_workqueue_attrs(wq->unbound_attrs);
  3366. kfree(wq);
  3367. return NULL;
  3368. err_destroy:
  3369. destroy_workqueue(wq);
  3370. return NULL;
  3371. }
  3372. EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
  3373. /**
  3374. * destroy_workqueue - safely terminate a workqueue
  3375. * @wq: target workqueue
  3376. *
  3377. * Safely destroy a workqueue. All work currently pending will be done first.
  3378. */
  3379. void destroy_workqueue(struct workqueue_struct *wq)
  3380. {
  3381. struct pool_workqueue *pwq;
  3382. /* drain it before proceeding with destruction */
  3383. drain_workqueue(wq);
  3384. /* sanity checks */
  3385. mutex_lock(&wq->mutex);
  3386. for_each_pwq(pwq, wq) {
  3387. int i;
  3388. for (i = 0; i < WORK_NR_COLORS; i++) {
  3389. if (WARN_ON(pwq->nr_in_flight[i])) {
  3390. mutex_unlock(&wq->mutex);
  3391. return;
  3392. }
  3393. }
  3394. if (WARN_ON(pwq->refcnt > 1) ||
  3395. WARN_ON(pwq->nr_active) ||
  3396. WARN_ON(!list_empty(&pwq->delayed_works))) {
  3397. mutex_unlock(&wq->mutex);
  3398. return;
  3399. }
  3400. }
  3401. mutex_unlock(&wq->mutex);
  3402. /*
  3403. * wq list is used to freeze wq, remove from list after
  3404. * flushing is complete in case freeze races us.
  3405. */
  3406. mutex_lock(&wq_pool_mutex);
  3407. list_del_init(&wq->list);
  3408. mutex_unlock(&wq_pool_mutex);
  3409. workqueue_sysfs_unregister(wq);
  3410. if (wq->rescuer) {
  3411. kthread_stop(wq->rescuer->task);
  3412. kfree(wq->rescuer);
  3413. wq->rescuer = NULL;
  3414. }
  3415. if (!(wq->flags & WQ_UNBOUND)) {
  3416. /*
  3417. * The base ref is never dropped on per-cpu pwqs. Directly
  3418. * free the pwqs and wq.
  3419. */
  3420. free_percpu(wq->cpu_pwqs);
  3421. kfree(wq);
  3422. } else {
  3423. /*
  3424. * We're the sole accessor of @wq at this point. Directly
  3425. * access the first pwq and put the base ref. @wq will be
  3426. * freed when the last pwq is released.
  3427. */
  3428. pwq = list_first_entry(&wq->pwqs, struct pool_workqueue,
  3429. pwqs_node);
  3430. put_pwq_unlocked(pwq);
  3431. }
  3432. }
  3433. EXPORT_SYMBOL_GPL(destroy_workqueue);
  3434. /**
  3435. * workqueue_set_max_active - adjust max_active of a workqueue
  3436. * @wq: target workqueue
  3437. * @max_active: new max_active value.
  3438. *
  3439. * Set max_active of @wq to @max_active.
  3440. *
  3441. * CONTEXT:
  3442. * Don't call from IRQ context.
  3443. */
  3444. void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
  3445. {
  3446. struct pool_workqueue *pwq;
  3447. /* disallow meddling with max_active for ordered workqueues */
  3448. if (WARN_ON(wq->flags & __WQ_ORDERED))
  3449. return;
  3450. max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
  3451. mutex_lock(&wq->mutex);
  3452. wq->saved_max_active = max_active;
  3453. for_each_pwq(pwq, wq)
  3454. pwq_adjust_max_active(pwq);
  3455. mutex_unlock(&wq->mutex);
  3456. }
  3457. EXPORT_SYMBOL_GPL(workqueue_set_max_active);
  3458. /**
  3459. * current_is_workqueue_rescuer - is %current workqueue rescuer?
  3460. *
  3461. * Determine whether %current is a workqueue rescuer. Can be used from
  3462. * work functions to determine whether it's being run off the rescuer task.
  3463. */
  3464. bool current_is_workqueue_rescuer(void)
  3465. {
  3466. struct worker *worker = current_wq_worker();
  3467. return worker && worker->rescue_wq;
  3468. }
  3469. /**
  3470. * workqueue_congested - test whether a workqueue is congested
  3471. * @cpu: CPU in question
  3472. * @wq: target workqueue
  3473. *
  3474. * Test whether @wq's cpu workqueue for @cpu is congested. There is
  3475. * no synchronization around this function and the test result is
  3476. * unreliable and only useful as advisory hints or for debugging.
  3477. *
  3478. * RETURNS:
  3479. * %true if congested, %false otherwise.
  3480. */
  3481. bool workqueue_congested(int cpu, struct workqueue_struct *wq)
  3482. {
  3483. struct pool_workqueue *pwq;
  3484. bool ret;
  3485. rcu_read_lock_sched();
  3486. if (!(wq->flags & WQ_UNBOUND))
  3487. pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
  3488. else
  3489. pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
  3490. ret = !list_empty(&pwq->delayed_works);
  3491. rcu_read_unlock_sched();
  3492. return ret;
  3493. }
  3494. EXPORT_SYMBOL_GPL(workqueue_congested);
  3495. /**
  3496. * work_busy - test whether a work is currently pending or running
  3497. * @work: the work to be tested
  3498. *
  3499. * Test whether @work is currently pending or running. There is no
  3500. * synchronization around this function and the test result is
  3501. * unreliable and only useful as advisory hints or for debugging.
  3502. *
  3503. * RETURNS:
  3504. * OR'd bitmask of WORK_BUSY_* bits.
  3505. */
  3506. unsigned int work_busy(struct work_struct *work)
  3507. {
  3508. struct worker_pool *pool;
  3509. unsigned long flags;
  3510. unsigned int ret = 0;
  3511. if (work_pending(work))
  3512. ret |= WORK_BUSY_PENDING;
  3513. local_irq_save(flags);
  3514. pool = get_work_pool(work);
  3515. if (pool) {
  3516. spin_lock(&pool->lock);
  3517. if (find_worker_executing_work(pool, work))
  3518. ret |= WORK_BUSY_RUNNING;
  3519. spin_unlock(&pool->lock);
  3520. }
  3521. local_irq_restore(flags);
  3522. return ret;
  3523. }
  3524. EXPORT_SYMBOL_GPL(work_busy);
  3525. /*
  3526. * CPU hotplug.
  3527. *
  3528. * There are two challenges in supporting CPU hotplug. Firstly, there
  3529. * are a lot of assumptions on strong associations among work, pwq and
  3530. * pool which make migrating pending and scheduled works very
  3531. * difficult to implement without impacting hot paths. Secondly,
  3532. * worker pools serve mix of short, long and very long running works making
  3533. * blocked draining impractical.
  3534. *
  3535. * This is solved by allowing the pools to be disassociated from the CPU
  3536. * running as an unbound one and allowing it to be reattached later if the
  3537. * cpu comes back online.
  3538. */
  3539. static void wq_unbind_fn(struct work_struct *work)
  3540. {
  3541. int cpu = smp_processor_id();
  3542. struct worker_pool *pool;
  3543. struct worker *worker;
  3544. int wi;
  3545. for_each_cpu_worker_pool(pool, cpu) {
  3546. WARN_ON_ONCE(cpu != smp_processor_id());
  3547. mutex_lock(&pool->manager_mutex);
  3548. spin_lock_irq(&pool->lock);
  3549. /*
  3550. * We've blocked all manager operations. Make all workers
  3551. * unbound and set DISASSOCIATED. Before this, all workers
  3552. * except for the ones which are still executing works from
  3553. * before the last CPU down must be on the cpu. After
  3554. * this, they may become diasporas.
  3555. */
  3556. for_each_pool_worker(worker, wi, pool)
  3557. worker->flags |= WORKER_UNBOUND;
  3558. pool->flags |= POOL_DISASSOCIATED;
  3559. spin_unlock_irq(&pool->lock);
  3560. mutex_unlock(&pool->manager_mutex);
  3561. }
  3562. /*
  3563. * Call schedule() so that we cross rq->lock and thus can guarantee
  3564. * sched callbacks see the %WORKER_UNBOUND flag. This is necessary
  3565. * as scheduler callbacks may be invoked from other cpus.
  3566. */
  3567. schedule();
  3568. /*
  3569. * Sched callbacks are disabled now. Zap nr_running. After this,
  3570. * nr_running stays zero and need_more_worker() and keep_working()
  3571. * are always true as long as the worklist is not empty. Pools on
  3572. * @cpu now behave as unbound (in terms of concurrency management)
  3573. * pools which are served by workers tied to the CPU.
  3574. *
  3575. * On return from this function, the current worker would trigger
  3576. * unbound chain execution of pending work items if other workers
  3577. * didn't already.
  3578. */
  3579. for_each_cpu_worker_pool(pool, cpu)
  3580. atomic_set(&pool->nr_running, 0);
  3581. }
  3582. /**
  3583. * rebind_workers - rebind all workers of a pool to the associated CPU
  3584. * @pool: pool of interest
  3585. *
  3586. * @pool->cpu is coming online. Rebind all workers to the CPU.
  3587. */
  3588. static void rebind_workers(struct worker_pool *pool)
  3589. {
  3590. struct worker *worker;
  3591. int wi;
  3592. lockdep_assert_held(&pool->manager_mutex);
  3593. /*
  3594. * Restore CPU affinity of all workers. As all idle workers should
  3595. * be on the run-queue of the associated CPU before any local
  3596. * wake-ups for concurrency management happen, restore CPU affinty
  3597. * of all workers first and then clear UNBOUND. As we're called
  3598. * from CPU_ONLINE, the following shouldn't fail.
  3599. */
  3600. for_each_pool_worker(worker, wi, pool)
  3601. WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
  3602. pool->attrs->cpumask) < 0);
  3603. spin_lock_irq(&pool->lock);
  3604. for_each_pool_worker(worker, wi, pool) {
  3605. unsigned int worker_flags = worker->flags;
  3606. /*
  3607. * A bound idle worker should actually be on the runqueue
  3608. * of the associated CPU for local wake-ups targeting it to
  3609. * work. Kick all idle workers so that they migrate to the
  3610. * associated CPU. Doing this in the same loop as
  3611. * replacing UNBOUND with REBOUND is safe as no worker will
  3612. * be bound before @pool->lock is released.
  3613. */
  3614. if (worker_flags & WORKER_IDLE)
  3615. wake_up_process(worker->task);
  3616. /*
  3617. * We want to clear UNBOUND but can't directly call
  3618. * worker_clr_flags() or adjust nr_running. Atomically
  3619. * replace UNBOUND with another NOT_RUNNING flag REBOUND.
  3620. * @worker will clear REBOUND using worker_clr_flags() when
  3621. * it initiates the next execution cycle thus restoring
  3622. * concurrency management. Note that when or whether
  3623. * @worker clears REBOUND doesn't affect correctness.
  3624. *
  3625. * ACCESS_ONCE() is necessary because @worker->flags may be
  3626. * tested without holding any lock in
  3627. * wq_worker_waking_up(). Without it, NOT_RUNNING test may
  3628. * fail incorrectly leading to premature concurrency
  3629. * management operations.
  3630. */
  3631. WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
  3632. worker_flags |= WORKER_REBOUND;
  3633. worker_flags &= ~WORKER_UNBOUND;
  3634. ACCESS_ONCE(worker->flags) = worker_flags;
  3635. }
  3636. spin_unlock_irq(&pool->lock);
  3637. }
  3638. /**
  3639. * restore_unbound_workers_cpumask - restore cpumask of unbound workers
  3640. * @pool: unbound pool of interest
  3641. * @cpu: the CPU which is coming up
  3642. *
  3643. * An unbound pool may end up with a cpumask which doesn't have any online
  3644. * CPUs. When a worker of such pool get scheduled, the scheduler resets
  3645. * its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
  3646. * online CPU before, cpus_allowed of all its workers should be restored.
  3647. */
  3648. static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
  3649. {
  3650. static cpumask_t cpumask;
  3651. struct worker *worker;
  3652. int wi;
  3653. lockdep_assert_held(&pool->manager_mutex);
  3654. /* is @cpu allowed for @pool? */
  3655. if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
  3656. return;
  3657. /* is @cpu the only online CPU? */
  3658. cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
  3659. if (cpumask_weight(&cpumask) != 1)
  3660. return;
  3661. /* as we're called from CPU_ONLINE, the following shouldn't fail */
  3662. for_each_pool_worker(worker, wi, pool)
  3663. WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
  3664. pool->attrs->cpumask) < 0);
  3665. }
  3666. /*
  3667. * Workqueues should be brought up before normal priority CPU notifiers.
  3668. * This will be registered high priority CPU notifier.
  3669. */
  3670. static int __cpuinit workqueue_cpu_up_callback(struct notifier_block *nfb,
  3671. unsigned long action,
  3672. void *hcpu)
  3673. {
  3674. int cpu = (unsigned long)hcpu;
  3675. struct worker_pool *pool;
  3676. int pi;
  3677. switch (action & ~CPU_TASKS_FROZEN) {
  3678. case CPU_UP_PREPARE:
  3679. for_each_cpu_worker_pool(pool, cpu) {
  3680. if (pool->nr_workers)
  3681. continue;
  3682. if (create_and_start_worker(pool) < 0)
  3683. return NOTIFY_BAD;
  3684. }
  3685. break;
  3686. case CPU_DOWN_FAILED:
  3687. case CPU_ONLINE:
  3688. mutex_lock(&wq_pool_mutex);
  3689. for_each_pool(pool, pi) {
  3690. mutex_lock(&pool->manager_mutex);
  3691. if (pool->cpu == cpu) {
  3692. spin_lock_irq(&pool->lock);
  3693. pool->flags &= ~POOL_DISASSOCIATED;
  3694. spin_unlock_irq(&pool->lock);
  3695. rebind_workers(pool);
  3696. } else if (pool->cpu < 0) {
  3697. restore_unbound_workers_cpumask(pool, cpu);
  3698. }
  3699. mutex_unlock(&pool->manager_mutex);
  3700. }
  3701. mutex_unlock(&wq_pool_mutex);
  3702. break;
  3703. }
  3704. return NOTIFY_OK;
  3705. }
  3706. /*
  3707. * Workqueues should be brought down after normal priority CPU notifiers.
  3708. * This will be registered as low priority CPU notifier.
  3709. */
  3710. static int __cpuinit workqueue_cpu_down_callback(struct notifier_block *nfb,
  3711. unsigned long action,
  3712. void *hcpu)
  3713. {
  3714. int cpu = (unsigned long)hcpu;
  3715. struct work_struct unbind_work;
  3716. switch (action & ~CPU_TASKS_FROZEN) {
  3717. case CPU_DOWN_PREPARE:
  3718. /* unbinding should happen on the local CPU */
  3719. INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
  3720. queue_work_on(cpu, system_highpri_wq, &unbind_work);
  3721. flush_work(&unbind_work);
  3722. break;
  3723. }
  3724. return NOTIFY_OK;
  3725. }
  3726. #ifdef CONFIG_SMP
  3727. struct work_for_cpu {
  3728. struct work_struct work;
  3729. long (*fn)(void *);
  3730. void *arg;
  3731. long ret;
  3732. };
  3733. static void work_for_cpu_fn(struct work_struct *work)
  3734. {
  3735. struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
  3736. wfc->ret = wfc->fn(wfc->arg);
  3737. }
  3738. /**
  3739. * work_on_cpu - run a function in user context on a particular cpu
  3740. * @cpu: the cpu to run on
  3741. * @fn: the function to run
  3742. * @arg: the function arg
  3743. *
  3744. * This will return the value @fn returns.
  3745. * It is up to the caller to ensure that the cpu doesn't go offline.
  3746. * The caller must not hold any locks which would prevent @fn from completing.
  3747. */
  3748. long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
  3749. {
  3750. struct work_for_cpu wfc = { .fn = fn, .arg = arg };
  3751. INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
  3752. schedule_work_on(cpu, &wfc.work);
  3753. flush_work(&wfc.work);
  3754. return wfc.ret;
  3755. }
  3756. EXPORT_SYMBOL_GPL(work_on_cpu);
  3757. #endif /* CONFIG_SMP */
  3758. #ifdef CONFIG_FREEZER
  3759. /**
  3760. * freeze_workqueues_begin - begin freezing workqueues
  3761. *
  3762. * Start freezing workqueues. After this function returns, all freezable
  3763. * workqueues will queue new works to their delayed_works list instead of
  3764. * pool->worklist.
  3765. *
  3766. * CONTEXT:
  3767. * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
  3768. */
  3769. void freeze_workqueues_begin(void)
  3770. {
  3771. struct worker_pool *pool;
  3772. struct workqueue_struct *wq;
  3773. struct pool_workqueue *pwq;
  3774. int pi;
  3775. mutex_lock(&wq_pool_mutex);
  3776. WARN_ON_ONCE(workqueue_freezing);
  3777. workqueue_freezing = true;
  3778. /* set FREEZING */
  3779. for_each_pool(pool, pi) {
  3780. spin_lock_irq(&pool->lock);
  3781. WARN_ON_ONCE(pool->flags & POOL_FREEZING);
  3782. pool->flags |= POOL_FREEZING;
  3783. spin_unlock_irq(&pool->lock);
  3784. }
  3785. list_for_each_entry(wq, &workqueues, list) {
  3786. mutex_lock(&wq->mutex);
  3787. for_each_pwq(pwq, wq)
  3788. pwq_adjust_max_active(pwq);
  3789. mutex_unlock(&wq->mutex);
  3790. }
  3791. mutex_unlock(&wq_pool_mutex);
  3792. }
  3793. /**
  3794. * freeze_workqueues_busy - are freezable workqueues still busy?
  3795. *
  3796. * Check whether freezing is complete. This function must be called
  3797. * between freeze_workqueues_begin() and thaw_workqueues().
  3798. *
  3799. * CONTEXT:
  3800. * Grabs and releases wq_pool_mutex.
  3801. *
  3802. * RETURNS:
  3803. * %true if some freezable workqueues are still busy. %false if freezing
  3804. * is complete.
  3805. */
  3806. bool freeze_workqueues_busy(void)
  3807. {
  3808. bool busy = false;
  3809. struct workqueue_struct *wq;
  3810. struct pool_workqueue *pwq;
  3811. mutex_lock(&wq_pool_mutex);
  3812. WARN_ON_ONCE(!workqueue_freezing);
  3813. list_for_each_entry(wq, &workqueues, list) {
  3814. if (!(wq->flags & WQ_FREEZABLE))
  3815. continue;
  3816. /*
  3817. * nr_active is monotonically decreasing. It's safe
  3818. * to peek without lock.
  3819. */
  3820. rcu_read_lock_sched();
  3821. for_each_pwq(pwq, wq) {
  3822. WARN_ON_ONCE(pwq->nr_active < 0);
  3823. if (pwq->nr_active) {
  3824. busy = true;
  3825. rcu_read_unlock_sched();
  3826. goto out_unlock;
  3827. }
  3828. }
  3829. rcu_read_unlock_sched();
  3830. }
  3831. out_unlock:
  3832. mutex_unlock(&wq_pool_mutex);
  3833. return busy;
  3834. }
  3835. /**
  3836. * thaw_workqueues - thaw workqueues
  3837. *
  3838. * Thaw workqueues. Normal queueing is restored and all collected
  3839. * frozen works are transferred to their respective pool worklists.
  3840. *
  3841. * CONTEXT:
  3842. * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
  3843. */
  3844. void thaw_workqueues(void)
  3845. {
  3846. struct workqueue_struct *wq;
  3847. struct pool_workqueue *pwq;
  3848. struct worker_pool *pool;
  3849. int pi;
  3850. mutex_lock(&wq_pool_mutex);
  3851. if (!workqueue_freezing)
  3852. goto out_unlock;
  3853. /* clear FREEZING */
  3854. for_each_pool(pool, pi) {
  3855. spin_lock_irq(&pool->lock);
  3856. WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
  3857. pool->flags &= ~POOL_FREEZING;
  3858. spin_unlock_irq(&pool->lock);
  3859. }
  3860. /* restore max_active and repopulate worklist */
  3861. list_for_each_entry(wq, &workqueues, list) {
  3862. mutex_lock(&wq->mutex);
  3863. for_each_pwq(pwq, wq)
  3864. pwq_adjust_max_active(pwq);
  3865. mutex_unlock(&wq->mutex);
  3866. }
  3867. workqueue_freezing = false;
  3868. out_unlock:
  3869. mutex_unlock(&wq_pool_mutex);
  3870. }
  3871. #endif /* CONFIG_FREEZER */
  3872. static void __init wq_numa_init(void)
  3873. {
  3874. cpumask_var_t *tbl;
  3875. int node, cpu;
  3876. /* determine NUMA pwq table len - highest node id + 1 */
  3877. for_each_node(node)
  3878. wq_numa_tbl_len = max(wq_numa_tbl_len, node + 1);
  3879. if (num_possible_nodes() <= 1)
  3880. return;
  3881. /*
  3882. * We want masks of possible CPUs of each node which isn't readily
  3883. * available. Build one from cpu_to_node() which should have been
  3884. * fully initialized by now.
  3885. */
  3886. tbl = kzalloc(wq_numa_tbl_len * sizeof(tbl[0]), GFP_KERNEL);
  3887. BUG_ON(!tbl);
  3888. for_each_node(node)
  3889. BUG_ON(!alloc_cpumask_var_node(&tbl[node], GFP_KERNEL, node));
  3890. for_each_possible_cpu(cpu) {
  3891. node = cpu_to_node(cpu);
  3892. if (WARN_ON(node == NUMA_NO_NODE)) {
  3893. pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
  3894. /* happens iff arch is bonkers, let's just proceed */
  3895. return;
  3896. }
  3897. cpumask_set_cpu(cpu, tbl[node]);
  3898. }
  3899. wq_numa_possible_cpumask = tbl;
  3900. wq_numa_enabled = true;
  3901. }
  3902. static int __init init_workqueues(void)
  3903. {
  3904. int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
  3905. int i, cpu;
  3906. /* make sure we have enough bits for OFFQ pool ID */
  3907. BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
  3908. WORK_CPU_END * NR_STD_WORKER_POOLS);
  3909. WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
  3910. pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
  3911. cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
  3912. hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
  3913. wq_numa_init();
  3914. /* initialize CPU pools */
  3915. for_each_possible_cpu(cpu) {
  3916. struct worker_pool *pool;
  3917. i = 0;
  3918. for_each_cpu_worker_pool(pool, cpu) {
  3919. BUG_ON(init_worker_pool(pool));
  3920. pool->cpu = cpu;
  3921. cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
  3922. pool->attrs->nice = std_nice[i++];
  3923. pool->node = cpu_to_node(cpu);
  3924. /* alloc pool ID */
  3925. mutex_lock(&wq_pool_mutex);
  3926. BUG_ON(worker_pool_assign_id(pool));
  3927. mutex_unlock(&wq_pool_mutex);
  3928. }
  3929. }
  3930. /* create the initial worker */
  3931. for_each_online_cpu(cpu) {
  3932. struct worker_pool *pool;
  3933. for_each_cpu_worker_pool(pool, cpu) {
  3934. pool->flags &= ~POOL_DISASSOCIATED;
  3935. BUG_ON(create_and_start_worker(pool) < 0);
  3936. }
  3937. }
  3938. /* create default unbound wq attrs */
  3939. for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
  3940. struct workqueue_attrs *attrs;
  3941. BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
  3942. attrs->nice = std_nice[i];
  3943. unbound_std_wq_attrs[i] = attrs;
  3944. }
  3945. system_wq = alloc_workqueue("events", 0, 0);
  3946. system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
  3947. system_long_wq = alloc_workqueue("events_long", 0, 0);
  3948. system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
  3949. WQ_UNBOUND_MAX_ACTIVE);
  3950. system_freezable_wq = alloc_workqueue("events_freezable",
  3951. WQ_FREEZABLE, 0);
  3952. BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
  3953. !system_unbound_wq || !system_freezable_wq);
  3954. return 0;
  3955. }
  3956. early_initcall(init_workqueues);