extent_io.c 81 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304
  1. #include <linux/bitops.h>
  2. #include <linux/slab.h>
  3. #include <linux/bio.h>
  4. #include <linux/mm.h>
  5. #include <linux/gfp.h>
  6. #include <linux/pagemap.h>
  7. #include <linux/page-flags.h>
  8. #include <linux/module.h>
  9. #include <linux/spinlock.h>
  10. #include <linux/blkdev.h>
  11. #include <linux/swap.h>
  12. #include <linux/version.h>
  13. #include <linux/writeback.h>
  14. #include <linux/pagevec.h>
  15. #include "extent_io.h"
  16. #include "extent_map.h"
  17. /* temporary define until extent_map moves out of btrfs */
  18. struct kmem_cache *btrfs_cache_create(const char *name, size_t size,
  19. unsigned long extra_flags,
  20. void (*ctor)(void *, struct kmem_cache *,
  21. unsigned long));
  22. static struct kmem_cache *extent_state_cache;
  23. static struct kmem_cache *extent_buffer_cache;
  24. static LIST_HEAD(buffers);
  25. static LIST_HEAD(states);
  26. #define BUFFER_LRU_MAX 64
  27. struct tree_entry {
  28. u64 start;
  29. u64 end;
  30. struct rb_node rb_node;
  31. };
  32. struct extent_page_data {
  33. struct bio *bio;
  34. struct extent_io_tree *tree;
  35. get_extent_t *get_extent;
  36. };
  37. int __init extent_io_init(void)
  38. {
  39. extent_state_cache = btrfs_cache_create("extent_state",
  40. sizeof(struct extent_state), 0,
  41. NULL);
  42. if (!extent_state_cache)
  43. return -ENOMEM;
  44. extent_buffer_cache = btrfs_cache_create("extent_buffers",
  45. sizeof(struct extent_buffer), 0,
  46. NULL);
  47. if (!extent_buffer_cache)
  48. goto free_state_cache;
  49. return 0;
  50. free_state_cache:
  51. kmem_cache_destroy(extent_state_cache);
  52. return -ENOMEM;
  53. }
  54. void extent_io_exit(void)
  55. {
  56. struct extent_state *state;
  57. while (!list_empty(&states)) {
  58. state = list_entry(states.next, struct extent_state, list);
  59. printk("state leak: start %Lu end %Lu state %lu in tree %p refs %d\n", state->start, state->end, state->state, state->tree, atomic_read(&state->refs));
  60. list_del(&state->list);
  61. kmem_cache_free(extent_state_cache, state);
  62. }
  63. if (extent_state_cache)
  64. kmem_cache_destroy(extent_state_cache);
  65. if (extent_buffer_cache)
  66. kmem_cache_destroy(extent_buffer_cache);
  67. }
  68. void extent_io_tree_init(struct extent_io_tree *tree,
  69. struct address_space *mapping, gfp_t mask)
  70. {
  71. tree->state.rb_node = NULL;
  72. tree->ops = NULL;
  73. tree->dirty_bytes = 0;
  74. spin_lock_init(&tree->lock);
  75. spin_lock_init(&tree->lru_lock);
  76. tree->mapping = mapping;
  77. INIT_LIST_HEAD(&tree->buffer_lru);
  78. tree->lru_size = 0;
  79. tree->last = NULL;
  80. }
  81. EXPORT_SYMBOL(extent_io_tree_init);
  82. void extent_io_tree_empty_lru(struct extent_io_tree *tree)
  83. {
  84. struct extent_buffer *eb;
  85. while(!list_empty(&tree->buffer_lru)) {
  86. eb = list_entry(tree->buffer_lru.next, struct extent_buffer,
  87. lru);
  88. list_del_init(&eb->lru);
  89. free_extent_buffer(eb);
  90. }
  91. }
  92. EXPORT_SYMBOL(extent_io_tree_empty_lru);
  93. struct extent_state *alloc_extent_state(gfp_t mask)
  94. {
  95. struct extent_state *state;
  96. state = kmem_cache_alloc(extent_state_cache, mask);
  97. if (!state || IS_ERR(state))
  98. return state;
  99. state->state = 0;
  100. state->private = 0;
  101. state->tree = NULL;
  102. atomic_set(&state->refs, 1);
  103. init_waitqueue_head(&state->wq);
  104. return state;
  105. }
  106. EXPORT_SYMBOL(alloc_extent_state);
  107. void free_extent_state(struct extent_state *state)
  108. {
  109. if (!state)
  110. return;
  111. if (atomic_dec_and_test(&state->refs)) {
  112. WARN_ON(state->tree);
  113. kmem_cache_free(extent_state_cache, state);
  114. }
  115. }
  116. EXPORT_SYMBOL(free_extent_state);
  117. static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
  118. struct rb_node *node)
  119. {
  120. struct rb_node ** p = &root->rb_node;
  121. struct rb_node * parent = NULL;
  122. struct tree_entry *entry;
  123. while(*p) {
  124. parent = *p;
  125. entry = rb_entry(parent, struct tree_entry, rb_node);
  126. if (offset < entry->start)
  127. p = &(*p)->rb_left;
  128. else if (offset > entry->end)
  129. p = &(*p)->rb_right;
  130. else
  131. return parent;
  132. }
  133. entry = rb_entry(node, struct tree_entry, rb_node);
  134. rb_link_node(node, parent, p);
  135. rb_insert_color(node, root);
  136. return NULL;
  137. }
  138. static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
  139. struct rb_node **prev_ret,
  140. struct rb_node **next_ret)
  141. {
  142. struct rb_root *root = &tree->state;
  143. struct rb_node * n = root->rb_node;
  144. struct rb_node *prev = NULL;
  145. struct rb_node *orig_prev = NULL;
  146. struct tree_entry *entry;
  147. struct tree_entry *prev_entry = NULL;
  148. if (tree->last) {
  149. struct extent_state *state;
  150. state = tree->last;
  151. if (state->start <= offset && offset <= state->end)
  152. return &tree->last->rb_node;
  153. }
  154. while(n) {
  155. entry = rb_entry(n, struct tree_entry, rb_node);
  156. prev = n;
  157. prev_entry = entry;
  158. if (offset < entry->start)
  159. n = n->rb_left;
  160. else if (offset > entry->end)
  161. n = n->rb_right;
  162. else {
  163. tree->last = rb_entry(n, struct extent_state, rb_node);
  164. return n;
  165. }
  166. }
  167. if (prev_ret) {
  168. orig_prev = prev;
  169. while(prev && offset > prev_entry->end) {
  170. prev = rb_next(prev);
  171. prev_entry = rb_entry(prev, struct tree_entry, rb_node);
  172. }
  173. *prev_ret = prev;
  174. prev = orig_prev;
  175. }
  176. if (next_ret) {
  177. prev_entry = rb_entry(prev, struct tree_entry, rb_node);
  178. while(prev && offset < prev_entry->start) {
  179. prev = rb_prev(prev);
  180. prev_entry = rb_entry(prev, struct tree_entry, rb_node);
  181. }
  182. *next_ret = prev;
  183. }
  184. return NULL;
  185. }
  186. static inline struct rb_node *tree_search(struct extent_io_tree *tree,
  187. u64 offset)
  188. {
  189. struct rb_node *prev = NULL;
  190. struct rb_node *ret;
  191. ret = __etree_search(tree, offset, &prev, NULL);
  192. if (!ret) {
  193. if (prev) {
  194. tree->last = rb_entry(prev, struct extent_state,
  195. rb_node);
  196. }
  197. return prev;
  198. }
  199. return ret;
  200. }
  201. /*
  202. * utility function to look for merge candidates inside a given range.
  203. * Any extents with matching state are merged together into a single
  204. * extent in the tree. Extents with EXTENT_IO in their state field
  205. * are not merged because the end_io handlers need to be able to do
  206. * operations on them without sleeping (or doing allocations/splits).
  207. *
  208. * This should be called with the tree lock held.
  209. */
  210. static int merge_state(struct extent_io_tree *tree,
  211. struct extent_state *state)
  212. {
  213. struct extent_state *other;
  214. struct rb_node *other_node;
  215. if (state->state & EXTENT_IOBITS)
  216. return 0;
  217. other_node = rb_prev(&state->rb_node);
  218. if (other_node) {
  219. other = rb_entry(other_node, struct extent_state, rb_node);
  220. if (other->end == state->start - 1 &&
  221. other->state == state->state) {
  222. state->start = other->start;
  223. other->tree = NULL;
  224. if (tree->last == other)
  225. tree->last = NULL;
  226. rb_erase(&other->rb_node, &tree->state);
  227. free_extent_state(other);
  228. }
  229. }
  230. other_node = rb_next(&state->rb_node);
  231. if (other_node) {
  232. other = rb_entry(other_node, struct extent_state, rb_node);
  233. if (other->start == state->end + 1 &&
  234. other->state == state->state) {
  235. other->start = state->start;
  236. state->tree = NULL;
  237. if (tree->last == state)
  238. tree->last = NULL;
  239. rb_erase(&state->rb_node, &tree->state);
  240. free_extent_state(state);
  241. }
  242. }
  243. return 0;
  244. }
  245. static void set_state_cb(struct extent_io_tree *tree,
  246. struct extent_state *state,
  247. unsigned long bits)
  248. {
  249. if (tree->ops && tree->ops->set_bit_hook) {
  250. tree->ops->set_bit_hook(tree->mapping->host, state->start,
  251. state->end, state->state, bits);
  252. }
  253. }
  254. static void clear_state_cb(struct extent_io_tree *tree,
  255. struct extent_state *state,
  256. unsigned long bits)
  257. {
  258. if (tree->ops && tree->ops->set_bit_hook) {
  259. tree->ops->clear_bit_hook(tree->mapping->host, state->start,
  260. state->end, state->state, bits);
  261. }
  262. }
  263. /*
  264. * insert an extent_state struct into the tree. 'bits' are set on the
  265. * struct before it is inserted.
  266. *
  267. * This may return -EEXIST if the extent is already there, in which case the
  268. * state struct is freed.
  269. *
  270. * The tree lock is not taken internally. This is a utility function and
  271. * probably isn't what you want to call (see set/clear_extent_bit).
  272. */
  273. static int insert_state(struct extent_io_tree *tree,
  274. struct extent_state *state, u64 start, u64 end,
  275. int bits)
  276. {
  277. struct rb_node *node;
  278. if (end < start) {
  279. printk("end < start %Lu %Lu\n", end, start);
  280. WARN_ON(1);
  281. }
  282. if (bits & EXTENT_DIRTY)
  283. tree->dirty_bytes += end - start + 1;
  284. set_state_cb(tree, state, bits);
  285. state->state |= bits;
  286. state->start = start;
  287. state->end = end;
  288. node = tree_insert(&tree->state, end, &state->rb_node);
  289. if (node) {
  290. struct extent_state *found;
  291. found = rb_entry(node, struct extent_state, rb_node);
  292. printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, start, end);
  293. free_extent_state(state);
  294. return -EEXIST;
  295. }
  296. state->tree = tree;
  297. tree->last = state;
  298. merge_state(tree, state);
  299. return 0;
  300. }
  301. /*
  302. * split a given extent state struct in two, inserting the preallocated
  303. * struct 'prealloc' as the newly created second half. 'split' indicates an
  304. * offset inside 'orig' where it should be split.
  305. *
  306. * Before calling,
  307. * the tree has 'orig' at [orig->start, orig->end]. After calling, there
  308. * are two extent state structs in the tree:
  309. * prealloc: [orig->start, split - 1]
  310. * orig: [ split, orig->end ]
  311. *
  312. * The tree locks are not taken by this function. They need to be held
  313. * by the caller.
  314. */
  315. static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
  316. struct extent_state *prealloc, u64 split)
  317. {
  318. struct rb_node *node;
  319. prealloc->start = orig->start;
  320. prealloc->end = split - 1;
  321. prealloc->state = orig->state;
  322. orig->start = split;
  323. node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
  324. if (node) {
  325. struct extent_state *found;
  326. found = rb_entry(node, struct extent_state, rb_node);
  327. printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, prealloc->start, prealloc->end);
  328. free_extent_state(prealloc);
  329. return -EEXIST;
  330. }
  331. prealloc->tree = tree;
  332. return 0;
  333. }
  334. /*
  335. * utility function to clear some bits in an extent state struct.
  336. * it will optionally wake up any one waiting on this state (wake == 1), or
  337. * forcibly remove the state from the tree (delete == 1).
  338. *
  339. * If no bits are set on the state struct after clearing things, the
  340. * struct is freed and removed from the tree
  341. */
  342. static int clear_state_bit(struct extent_io_tree *tree,
  343. struct extent_state *state, int bits, int wake,
  344. int delete)
  345. {
  346. int ret = state->state & bits;
  347. if ((bits & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
  348. u64 range = state->end - state->start + 1;
  349. WARN_ON(range > tree->dirty_bytes);
  350. tree->dirty_bytes -= range;
  351. }
  352. clear_state_cb(tree, state, bits);
  353. state->state &= ~bits;
  354. if (wake)
  355. wake_up(&state->wq);
  356. if (delete || state->state == 0) {
  357. if (state->tree) {
  358. clear_state_cb(tree, state, state->state);
  359. if (tree->last == state)
  360. tree->last = NULL;
  361. rb_erase(&state->rb_node, &tree->state);
  362. state->tree = NULL;
  363. free_extent_state(state);
  364. } else {
  365. WARN_ON(1);
  366. }
  367. } else {
  368. merge_state(tree, state);
  369. }
  370. return ret;
  371. }
  372. /*
  373. * clear some bits on a range in the tree. This may require splitting
  374. * or inserting elements in the tree, so the gfp mask is used to
  375. * indicate which allocations or sleeping are allowed.
  376. *
  377. * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
  378. * the given range from the tree regardless of state (ie for truncate).
  379. *
  380. * the range [start, end] is inclusive.
  381. *
  382. * This takes the tree lock, and returns < 0 on error, > 0 if any of the
  383. * bits were already set, or zero if none of the bits were already set.
  384. */
  385. int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
  386. int bits, int wake, int delete, gfp_t mask)
  387. {
  388. struct extent_state *state;
  389. struct extent_state *prealloc = NULL;
  390. struct rb_node *node;
  391. unsigned long flags;
  392. int err;
  393. int set = 0;
  394. again:
  395. if (!prealloc && (mask & __GFP_WAIT)) {
  396. prealloc = alloc_extent_state(mask);
  397. if (!prealloc)
  398. return -ENOMEM;
  399. }
  400. spin_lock_irqsave(&tree->lock, flags);
  401. /*
  402. * this search will find the extents that end after
  403. * our range starts
  404. */
  405. node = tree_search(tree, start);
  406. if (!node)
  407. goto out;
  408. state = rb_entry(node, struct extent_state, rb_node);
  409. if (state->start > end)
  410. goto out;
  411. WARN_ON(state->end < start);
  412. /*
  413. * | ---- desired range ---- |
  414. * | state | or
  415. * | ------------- state -------------- |
  416. *
  417. * We need to split the extent we found, and may flip
  418. * bits on second half.
  419. *
  420. * If the extent we found extends past our range, we
  421. * just split and search again. It'll get split again
  422. * the next time though.
  423. *
  424. * If the extent we found is inside our range, we clear
  425. * the desired bit on it.
  426. */
  427. if (state->start < start) {
  428. if (!prealloc)
  429. prealloc = alloc_extent_state(GFP_ATOMIC);
  430. err = split_state(tree, state, prealloc, start);
  431. BUG_ON(err == -EEXIST);
  432. prealloc = NULL;
  433. if (err)
  434. goto out;
  435. if (state->end <= end) {
  436. start = state->end + 1;
  437. set |= clear_state_bit(tree, state, bits,
  438. wake, delete);
  439. } else {
  440. start = state->start;
  441. }
  442. goto search_again;
  443. }
  444. /*
  445. * | ---- desired range ---- |
  446. * | state |
  447. * We need to split the extent, and clear the bit
  448. * on the first half
  449. */
  450. if (state->start <= end && state->end > end) {
  451. if (!prealloc)
  452. prealloc = alloc_extent_state(GFP_ATOMIC);
  453. err = split_state(tree, state, prealloc, end + 1);
  454. BUG_ON(err == -EEXIST);
  455. if (wake)
  456. wake_up(&state->wq);
  457. set |= clear_state_bit(tree, prealloc, bits,
  458. wake, delete);
  459. prealloc = NULL;
  460. goto out;
  461. }
  462. start = state->end + 1;
  463. set |= clear_state_bit(tree, state, bits, wake, delete);
  464. goto search_again;
  465. out:
  466. spin_unlock_irqrestore(&tree->lock, flags);
  467. if (prealloc)
  468. free_extent_state(prealloc);
  469. return set;
  470. search_again:
  471. if (start > end)
  472. goto out;
  473. spin_unlock_irqrestore(&tree->lock, flags);
  474. if (mask & __GFP_WAIT)
  475. cond_resched();
  476. goto again;
  477. }
  478. EXPORT_SYMBOL(clear_extent_bit);
  479. static int wait_on_state(struct extent_io_tree *tree,
  480. struct extent_state *state)
  481. {
  482. DEFINE_WAIT(wait);
  483. prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
  484. spin_unlock_irq(&tree->lock);
  485. schedule();
  486. spin_lock_irq(&tree->lock);
  487. finish_wait(&state->wq, &wait);
  488. return 0;
  489. }
  490. /*
  491. * waits for one or more bits to clear on a range in the state tree.
  492. * The range [start, end] is inclusive.
  493. * The tree lock is taken by this function
  494. */
  495. int wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits)
  496. {
  497. struct extent_state *state;
  498. struct rb_node *node;
  499. spin_lock_irq(&tree->lock);
  500. again:
  501. while (1) {
  502. /*
  503. * this search will find all the extents that end after
  504. * our range starts
  505. */
  506. node = tree_search(tree, start);
  507. if (!node)
  508. break;
  509. state = rb_entry(node, struct extent_state, rb_node);
  510. if (state->start > end)
  511. goto out;
  512. if (state->state & bits) {
  513. start = state->start;
  514. atomic_inc(&state->refs);
  515. wait_on_state(tree, state);
  516. free_extent_state(state);
  517. goto again;
  518. }
  519. start = state->end + 1;
  520. if (start > end)
  521. break;
  522. if (need_resched()) {
  523. spin_unlock_irq(&tree->lock);
  524. cond_resched();
  525. spin_lock_irq(&tree->lock);
  526. }
  527. }
  528. out:
  529. spin_unlock_irq(&tree->lock);
  530. return 0;
  531. }
  532. EXPORT_SYMBOL(wait_extent_bit);
  533. static void set_state_bits(struct extent_io_tree *tree,
  534. struct extent_state *state,
  535. int bits)
  536. {
  537. if ((bits & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
  538. u64 range = state->end - state->start + 1;
  539. tree->dirty_bytes += range;
  540. }
  541. set_state_cb(tree, state, bits);
  542. state->state |= bits;
  543. }
  544. /*
  545. * set some bits on a range in the tree. This may require allocations
  546. * or sleeping, so the gfp mask is used to indicate what is allowed.
  547. *
  548. * If 'exclusive' == 1, this will fail with -EEXIST if some part of the
  549. * range already has the desired bits set. The start of the existing
  550. * range is returned in failed_start in this case.
  551. *
  552. * [start, end] is inclusive
  553. * This takes the tree lock.
  554. */
  555. int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits,
  556. int exclusive, u64 *failed_start, gfp_t mask)
  557. {
  558. struct extent_state *state;
  559. struct extent_state *prealloc = NULL;
  560. struct rb_node *node;
  561. unsigned long flags;
  562. int err = 0;
  563. int set;
  564. u64 last_start;
  565. u64 last_end;
  566. again:
  567. if (!prealloc && (mask & __GFP_WAIT)) {
  568. prealloc = alloc_extent_state(mask);
  569. if (!prealloc)
  570. return -ENOMEM;
  571. }
  572. spin_lock_irqsave(&tree->lock, flags);
  573. /*
  574. * this search will find all the extents that end after
  575. * our range starts.
  576. */
  577. node = tree_search(tree, start);
  578. if (!node) {
  579. err = insert_state(tree, prealloc, start, end, bits);
  580. prealloc = NULL;
  581. BUG_ON(err == -EEXIST);
  582. goto out;
  583. }
  584. state = rb_entry(node, struct extent_state, rb_node);
  585. last_start = state->start;
  586. last_end = state->end;
  587. /*
  588. * | ---- desired range ---- |
  589. * | state |
  590. *
  591. * Just lock what we found and keep going
  592. */
  593. if (state->start == start && state->end <= end) {
  594. set = state->state & bits;
  595. if (set && exclusive) {
  596. *failed_start = state->start;
  597. err = -EEXIST;
  598. goto out;
  599. }
  600. set_state_bits(tree, state, bits);
  601. start = state->end + 1;
  602. merge_state(tree, state);
  603. goto search_again;
  604. }
  605. /*
  606. * | ---- desired range ---- |
  607. * | state |
  608. * or
  609. * | ------------- state -------------- |
  610. *
  611. * We need to split the extent we found, and may flip bits on
  612. * second half.
  613. *
  614. * If the extent we found extends past our
  615. * range, we just split and search again. It'll get split
  616. * again the next time though.
  617. *
  618. * If the extent we found is inside our range, we set the
  619. * desired bit on it.
  620. */
  621. if (state->start < start) {
  622. set = state->state & bits;
  623. if (exclusive && set) {
  624. *failed_start = start;
  625. err = -EEXIST;
  626. goto out;
  627. }
  628. err = split_state(tree, state, prealloc, start);
  629. BUG_ON(err == -EEXIST);
  630. prealloc = NULL;
  631. if (err)
  632. goto out;
  633. if (state->end <= end) {
  634. set_state_bits(tree, state, bits);
  635. start = state->end + 1;
  636. merge_state(tree, state);
  637. } else {
  638. start = state->start;
  639. }
  640. goto search_again;
  641. }
  642. /*
  643. * | ---- desired range ---- |
  644. * | state | or | state |
  645. *
  646. * There's a hole, we need to insert something in it and
  647. * ignore the extent we found.
  648. */
  649. if (state->start > start) {
  650. u64 this_end;
  651. if (end < last_start)
  652. this_end = end;
  653. else
  654. this_end = last_start -1;
  655. err = insert_state(tree, prealloc, start, this_end,
  656. bits);
  657. prealloc = NULL;
  658. BUG_ON(err == -EEXIST);
  659. if (err)
  660. goto out;
  661. start = this_end + 1;
  662. goto search_again;
  663. }
  664. /*
  665. * | ---- desired range ---- |
  666. * | state |
  667. * We need to split the extent, and set the bit
  668. * on the first half
  669. */
  670. if (state->start <= end && state->end > end) {
  671. set = state->state & bits;
  672. if (exclusive && set) {
  673. *failed_start = start;
  674. err = -EEXIST;
  675. goto out;
  676. }
  677. err = split_state(tree, state, prealloc, end + 1);
  678. BUG_ON(err == -EEXIST);
  679. set_state_bits(tree, prealloc, bits);
  680. merge_state(tree, prealloc);
  681. prealloc = NULL;
  682. goto out;
  683. }
  684. goto search_again;
  685. out:
  686. spin_unlock_irqrestore(&tree->lock, flags);
  687. if (prealloc)
  688. free_extent_state(prealloc);
  689. return err;
  690. search_again:
  691. if (start > end)
  692. goto out;
  693. spin_unlock_irqrestore(&tree->lock, flags);
  694. if (mask & __GFP_WAIT)
  695. cond_resched();
  696. goto again;
  697. }
  698. EXPORT_SYMBOL(set_extent_bit);
  699. /* wrappers around set/clear extent bit */
  700. int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
  701. gfp_t mask)
  702. {
  703. return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL,
  704. mask);
  705. }
  706. EXPORT_SYMBOL(set_extent_dirty);
  707. int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
  708. int bits, gfp_t mask)
  709. {
  710. return set_extent_bit(tree, start, end, bits, 0, NULL,
  711. mask);
  712. }
  713. EXPORT_SYMBOL(set_extent_bits);
  714. int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
  715. int bits, gfp_t mask)
  716. {
  717. return clear_extent_bit(tree, start, end, bits, 0, 0, mask);
  718. }
  719. EXPORT_SYMBOL(clear_extent_bits);
  720. int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
  721. gfp_t mask)
  722. {
  723. return set_extent_bit(tree, start, end,
  724. EXTENT_DELALLOC | EXTENT_DIRTY, 0, NULL,
  725. mask);
  726. }
  727. EXPORT_SYMBOL(set_extent_delalloc);
  728. int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
  729. gfp_t mask)
  730. {
  731. return clear_extent_bit(tree, start, end,
  732. EXTENT_DIRTY | EXTENT_DELALLOC, 0, 0, mask);
  733. }
  734. EXPORT_SYMBOL(clear_extent_dirty);
  735. int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
  736. gfp_t mask)
  737. {
  738. return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL,
  739. mask);
  740. }
  741. EXPORT_SYMBOL(set_extent_new);
  742. int clear_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
  743. gfp_t mask)
  744. {
  745. return clear_extent_bit(tree, start, end, EXTENT_NEW, 0, 0, mask);
  746. }
  747. EXPORT_SYMBOL(clear_extent_new);
  748. int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
  749. gfp_t mask)
  750. {
  751. return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, NULL,
  752. mask);
  753. }
  754. EXPORT_SYMBOL(set_extent_uptodate);
  755. int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
  756. gfp_t mask)
  757. {
  758. return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, mask);
  759. }
  760. EXPORT_SYMBOL(clear_extent_uptodate);
  761. int set_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end,
  762. gfp_t mask)
  763. {
  764. return set_extent_bit(tree, start, end, EXTENT_WRITEBACK,
  765. 0, NULL, mask);
  766. }
  767. EXPORT_SYMBOL(set_extent_writeback);
  768. int clear_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end,
  769. gfp_t mask)
  770. {
  771. return clear_extent_bit(tree, start, end, EXTENT_WRITEBACK, 1, 0, mask);
  772. }
  773. EXPORT_SYMBOL(clear_extent_writeback);
  774. int wait_on_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end)
  775. {
  776. return wait_extent_bit(tree, start, end, EXTENT_WRITEBACK);
  777. }
  778. EXPORT_SYMBOL(wait_on_extent_writeback);
  779. int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask)
  780. {
  781. int err;
  782. u64 failed_start;
  783. while (1) {
  784. err = set_extent_bit(tree, start, end, EXTENT_LOCKED, 1,
  785. &failed_start, mask);
  786. if (err == -EEXIST && (mask & __GFP_WAIT)) {
  787. wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
  788. start = failed_start;
  789. } else {
  790. break;
  791. }
  792. WARN_ON(start > end);
  793. }
  794. return err;
  795. }
  796. EXPORT_SYMBOL(lock_extent);
  797. int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end,
  798. gfp_t mask)
  799. {
  800. return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, mask);
  801. }
  802. EXPORT_SYMBOL(unlock_extent);
  803. /*
  804. * helper function to set pages and extents in the tree dirty
  805. */
  806. int set_range_dirty(struct extent_io_tree *tree, u64 start, u64 end)
  807. {
  808. unsigned long index = start >> PAGE_CACHE_SHIFT;
  809. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  810. struct page *page;
  811. while (index <= end_index) {
  812. page = find_get_page(tree->mapping, index);
  813. BUG_ON(!page);
  814. __set_page_dirty_nobuffers(page);
  815. page_cache_release(page);
  816. index++;
  817. }
  818. set_extent_dirty(tree, start, end, GFP_NOFS);
  819. return 0;
  820. }
  821. EXPORT_SYMBOL(set_range_dirty);
  822. /*
  823. * helper function to set both pages and extents in the tree writeback
  824. */
  825. int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
  826. {
  827. unsigned long index = start >> PAGE_CACHE_SHIFT;
  828. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  829. struct page *page;
  830. while (index <= end_index) {
  831. page = find_get_page(tree->mapping, index);
  832. BUG_ON(!page);
  833. set_page_writeback(page);
  834. page_cache_release(page);
  835. index++;
  836. }
  837. set_extent_writeback(tree, start, end, GFP_NOFS);
  838. return 0;
  839. }
  840. EXPORT_SYMBOL(set_range_writeback);
  841. int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
  842. u64 *start_ret, u64 *end_ret, int bits)
  843. {
  844. struct rb_node *node;
  845. struct extent_state *state;
  846. int ret = 1;
  847. spin_lock_irq(&tree->lock);
  848. /*
  849. * this search will find all the extents that end after
  850. * our range starts.
  851. */
  852. node = tree_search(tree, start);
  853. if (!node || IS_ERR(node)) {
  854. goto out;
  855. }
  856. while(1) {
  857. state = rb_entry(node, struct extent_state, rb_node);
  858. if (state->end >= start && (state->state & bits)) {
  859. *start_ret = state->start;
  860. *end_ret = state->end;
  861. ret = 0;
  862. break;
  863. }
  864. node = rb_next(node);
  865. if (!node)
  866. break;
  867. }
  868. out:
  869. spin_unlock_irq(&tree->lock);
  870. return ret;
  871. }
  872. EXPORT_SYMBOL(find_first_extent_bit);
  873. u64 find_lock_delalloc_range(struct extent_io_tree *tree,
  874. u64 *start, u64 *end, u64 max_bytes)
  875. {
  876. struct rb_node *node;
  877. struct extent_state *state;
  878. u64 cur_start = *start;
  879. u64 found = 0;
  880. u64 total_bytes = 0;
  881. spin_lock_irq(&tree->lock);
  882. /*
  883. * this search will find all the extents that end after
  884. * our range starts.
  885. */
  886. search_again:
  887. node = tree_search(tree, cur_start);
  888. if (!node || IS_ERR(node)) {
  889. *end = (u64)-1;
  890. goto out;
  891. }
  892. while(1) {
  893. state = rb_entry(node, struct extent_state, rb_node);
  894. if (found && state->start != cur_start) {
  895. goto out;
  896. }
  897. if (!(state->state & EXTENT_DELALLOC)) {
  898. if (!found)
  899. *end = state->end;
  900. goto out;
  901. }
  902. if (!found) {
  903. struct extent_state *prev_state;
  904. struct rb_node *prev_node = node;
  905. while(1) {
  906. prev_node = rb_prev(prev_node);
  907. if (!prev_node)
  908. break;
  909. prev_state = rb_entry(prev_node,
  910. struct extent_state,
  911. rb_node);
  912. if (!(prev_state->state & EXTENT_DELALLOC))
  913. break;
  914. state = prev_state;
  915. node = prev_node;
  916. }
  917. }
  918. if (state->state & EXTENT_LOCKED) {
  919. DEFINE_WAIT(wait);
  920. atomic_inc(&state->refs);
  921. prepare_to_wait(&state->wq, &wait,
  922. TASK_UNINTERRUPTIBLE);
  923. spin_unlock_irq(&tree->lock);
  924. schedule();
  925. spin_lock_irq(&tree->lock);
  926. finish_wait(&state->wq, &wait);
  927. free_extent_state(state);
  928. goto search_again;
  929. }
  930. set_state_cb(tree, state, EXTENT_LOCKED);
  931. state->state |= EXTENT_LOCKED;
  932. if (!found)
  933. *start = state->start;
  934. found++;
  935. *end = state->end;
  936. cur_start = state->end + 1;
  937. node = rb_next(node);
  938. if (!node)
  939. break;
  940. total_bytes += state->end - state->start + 1;
  941. if (total_bytes >= max_bytes)
  942. break;
  943. }
  944. out:
  945. spin_unlock_irq(&tree->lock);
  946. return found;
  947. }
  948. u64 count_range_bits(struct extent_io_tree *tree,
  949. u64 *start, u64 search_end, u64 max_bytes,
  950. unsigned long bits)
  951. {
  952. struct rb_node *node;
  953. struct extent_state *state;
  954. u64 cur_start = *start;
  955. u64 total_bytes = 0;
  956. int found = 0;
  957. if (search_end <= cur_start) {
  958. printk("search_end %Lu start %Lu\n", search_end, cur_start);
  959. WARN_ON(1);
  960. return 0;
  961. }
  962. spin_lock_irq(&tree->lock);
  963. if (cur_start == 0 && bits == EXTENT_DIRTY) {
  964. total_bytes = tree->dirty_bytes;
  965. goto out;
  966. }
  967. /*
  968. * this search will find all the extents that end after
  969. * our range starts.
  970. */
  971. node = tree_search(tree, cur_start);
  972. if (!node || IS_ERR(node)) {
  973. goto out;
  974. }
  975. while(1) {
  976. state = rb_entry(node, struct extent_state, rb_node);
  977. if (state->start > search_end)
  978. break;
  979. if (state->end >= cur_start && (state->state & bits)) {
  980. total_bytes += min(search_end, state->end) + 1 -
  981. max(cur_start, state->start);
  982. if (total_bytes >= max_bytes)
  983. break;
  984. if (!found) {
  985. *start = state->start;
  986. found = 1;
  987. }
  988. }
  989. node = rb_next(node);
  990. if (!node)
  991. break;
  992. }
  993. out:
  994. spin_unlock_irq(&tree->lock);
  995. return total_bytes;
  996. }
  997. /*
  998. * helper function to lock both pages and extents in the tree.
  999. * pages must be locked first.
  1000. */
  1001. int lock_range(struct extent_io_tree *tree, u64 start, u64 end)
  1002. {
  1003. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1004. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1005. struct page *page;
  1006. int err;
  1007. while (index <= end_index) {
  1008. page = grab_cache_page(tree->mapping, index);
  1009. if (!page) {
  1010. err = -ENOMEM;
  1011. goto failed;
  1012. }
  1013. if (IS_ERR(page)) {
  1014. err = PTR_ERR(page);
  1015. goto failed;
  1016. }
  1017. index++;
  1018. }
  1019. lock_extent(tree, start, end, GFP_NOFS);
  1020. return 0;
  1021. failed:
  1022. /*
  1023. * we failed above in getting the page at 'index', so we undo here
  1024. * up to but not including the page at 'index'
  1025. */
  1026. end_index = index;
  1027. index = start >> PAGE_CACHE_SHIFT;
  1028. while (index < end_index) {
  1029. page = find_get_page(tree->mapping, index);
  1030. unlock_page(page);
  1031. page_cache_release(page);
  1032. index++;
  1033. }
  1034. return err;
  1035. }
  1036. EXPORT_SYMBOL(lock_range);
  1037. /*
  1038. * helper function to unlock both pages and extents in the tree.
  1039. */
  1040. int unlock_range(struct extent_io_tree *tree, u64 start, u64 end)
  1041. {
  1042. unsigned long index = start >> PAGE_CACHE_SHIFT;
  1043. unsigned long end_index = end >> PAGE_CACHE_SHIFT;
  1044. struct page *page;
  1045. while (index <= end_index) {
  1046. page = find_get_page(tree->mapping, index);
  1047. unlock_page(page);
  1048. page_cache_release(page);
  1049. index++;
  1050. }
  1051. unlock_extent(tree, start, end, GFP_NOFS);
  1052. return 0;
  1053. }
  1054. EXPORT_SYMBOL(unlock_range);
  1055. int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
  1056. {
  1057. struct rb_node *node;
  1058. struct extent_state *state;
  1059. int ret = 0;
  1060. spin_lock_irq(&tree->lock);
  1061. /*
  1062. * this search will find all the extents that end after
  1063. * our range starts.
  1064. */
  1065. node = tree_search(tree, start);
  1066. if (!node || IS_ERR(node)) {
  1067. ret = -ENOENT;
  1068. goto out;
  1069. }
  1070. state = rb_entry(node, struct extent_state, rb_node);
  1071. if (state->start != start) {
  1072. ret = -ENOENT;
  1073. goto out;
  1074. }
  1075. state->private = private;
  1076. out:
  1077. spin_unlock_irq(&tree->lock);
  1078. return ret;
  1079. }
  1080. int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
  1081. {
  1082. struct rb_node *node;
  1083. struct extent_state *state;
  1084. int ret = 0;
  1085. spin_lock_irq(&tree->lock);
  1086. /*
  1087. * this search will find all the extents that end after
  1088. * our range starts.
  1089. */
  1090. node = tree_search(tree, start);
  1091. if (!node || IS_ERR(node)) {
  1092. ret = -ENOENT;
  1093. goto out;
  1094. }
  1095. state = rb_entry(node, struct extent_state, rb_node);
  1096. if (state->start != start) {
  1097. ret = -ENOENT;
  1098. goto out;
  1099. }
  1100. *private = state->private;
  1101. out:
  1102. spin_unlock_irq(&tree->lock);
  1103. return ret;
  1104. }
  1105. /*
  1106. * searches a range in the state tree for a given mask.
  1107. * If 'filled' == 1, this returns 1 only if every extent in the tree
  1108. * has the bits set. Otherwise, 1 is returned if any bit in the
  1109. * range is found set.
  1110. */
  1111. int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
  1112. int bits, int filled)
  1113. {
  1114. struct extent_state *state = NULL;
  1115. struct rb_node *node;
  1116. int bitset = 0;
  1117. unsigned long flags;
  1118. spin_lock_irqsave(&tree->lock, flags);
  1119. node = tree_search(tree, start);
  1120. while (node && start <= end) {
  1121. state = rb_entry(node, struct extent_state, rb_node);
  1122. if (filled && state->start > start) {
  1123. bitset = 0;
  1124. break;
  1125. }
  1126. if (state->start > end)
  1127. break;
  1128. if (state->state & bits) {
  1129. bitset = 1;
  1130. if (!filled)
  1131. break;
  1132. } else if (filled) {
  1133. bitset = 0;
  1134. break;
  1135. }
  1136. start = state->end + 1;
  1137. if (start > end)
  1138. break;
  1139. node = rb_next(node);
  1140. if (!node) {
  1141. if (filled)
  1142. bitset = 0;
  1143. break;
  1144. }
  1145. }
  1146. spin_unlock_irqrestore(&tree->lock, flags);
  1147. return bitset;
  1148. }
  1149. EXPORT_SYMBOL(test_range_bit);
  1150. /*
  1151. * helper function to set a given page up to date if all the
  1152. * extents in the tree for that page are up to date
  1153. */
  1154. static int check_page_uptodate(struct extent_io_tree *tree,
  1155. struct page *page)
  1156. {
  1157. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1158. u64 end = start + PAGE_CACHE_SIZE - 1;
  1159. if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1))
  1160. SetPageUptodate(page);
  1161. return 0;
  1162. }
  1163. /*
  1164. * helper function to unlock a page if all the extents in the tree
  1165. * for that page are unlocked
  1166. */
  1167. static int check_page_locked(struct extent_io_tree *tree,
  1168. struct page *page)
  1169. {
  1170. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1171. u64 end = start + PAGE_CACHE_SIZE - 1;
  1172. if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0))
  1173. unlock_page(page);
  1174. return 0;
  1175. }
  1176. /*
  1177. * helper function to end page writeback if all the extents
  1178. * in the tree for that page are done with writeback
  1179. */
  1180. static int check_page_writeback(struct extent_io_tree *tree,
  1181. struct page *page)
  1182. {
  1183. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1184. u64 end = start + PAGE_CACHE_SIZE - 1;
  1185. if (!test_range_bit(tree, start, end, EXTENT_WRITEBACK, 0))
  1186. end_page_writeback(page);
  1187. return 0;
  1188. }
  1189. /* lots and lots of room for performance fixes in the end_bio funcs */
  1190. /*
  1191. * after a writepage IO is done, we need to:
  1192. * clear the uptodate bits on error
  1193. * clear the writeback bits in the extent tree for this IO
  1194. * end_page_writeback if the page has no more pending IO
  1195. *
  1196. * Scheduling is not allowed, so the extent state tree is expected
  1197. * to have one and only one object corresponding to this IO.
  1198. */
  1199. #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
  1200. static void end_bio_extent_writepage(struct bio *bio, int err)
  1201. #else
  1202. static int end_bio_extent_writepage(struct bio *bio,
  1203. unsigned int bytes_done, int err)
  1204. #endif
  1205. {
  1206. const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  1207. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1208. struct extent_state *state = bio->bi_private;
  1209. struct extent_io_tree *tree = state->tree;
  1210. struct rb_node *node;
  1211. u64 start;
  1212. u64 end;
  1213. u64 cur;
  1214. int whole_page;
  1215. unsigned long flags;
  1216. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1217. if (bio->bi_size)
  1218. return 1;
  1219. #endif
  1220. do {
  1221. struct page *page = bvec->bv_page;
  1222. start = ((u64)page->index << PAGE_CACHE_SHIFT) +
  1223. bvec->bv_offset;
  1224. end = start + bvec->bv_len - 1;
  1225. if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
  1226. whole_page = 1;
  1227. else
  1228. whole_page = 0;
  1229. if (--bvec >= bio->bi_io_vec)
  1230. prefetchw(&bvec->bv_page->flags);
  1231. if (!uptodate) {
  1232. clear_extent_uptodate(tree, start, end, GFP_ATOMIC);
  1233. ClearPageUptodate(page);
  1234. SetPageError(page);
  1235. }
  1236. if (tree->ops && tree->ops->writepage_end_io_hook) {
  1237. tree->ops->writepage_end_io_hook(page, start, end,
  1238. state);
  1239. }
  1240. /*
  1241. * bios can get merged in funny ways, and so we need to
  1242. * be careful with the state variable. We know the
  1243. * state won't be merged with others because it has
  1244. * WRITEBACK set, but we can't be sure each biovec is
  1245. * sequential in the file. So, if our cached state
  1246. * doesn't match the expected end, search the tree
  1247. * for the correct one.
  1248. */
  1249. spin_lock_irqsave(&tree->lock, flags);
  1250. if (!state || state->end != end) {
  1251. state = NULL;
  1252. node = __etree_search(tree, start, NULL, NULL);
  1253. if (node) {
  1254. state = rb_entry(node, struct extent_state,
  1255. rb_node);
  1256. if (state->end != end ||
  1257. !(state->state & EXTENT_WRITEBACK))
  1258. state = NULL;
  1259. }
  1260. if (!state) {
  1261. spin_unlock_irqrestore(&tree->lock, flags);
  1262. clear_extent_writeback(tree, start,
  1263. end, GFP_ATOMIC);
  1264. goto next_io;
  1265. }
  1266. }
  1267. cur = end;
  1268. while(1) {
  1269. struct extent_state *clear = state;
  1270. cur = state->start;
  1271. node = rb_prev(&state->rb_node);
  1272. if (node) {
  1273. state = rb_entry(node,
  1274. struct extent_state,
  1275. rb_node);
  1276. } else {
  1277. state = NULL;
  1278. }
  1279. clear_state_bit(tree, clear, EXTENT_WRITEBACK,
  1280. 1, 0);
  1281. if (cur == start)
  1282. break;
  1283. if (cur < start) {
  1284. WARN_ON(1);
  1285. break;
  1286. }
  1287. if (!node)
  1288. break;
  1289. }
  1290. /* before releasing the lock, make sure the next state
  1291. * variable has the expected bits set and corresponds
  1292. * to the correct offsets in the file
  1293. */
  1294. if (state && (state->end + 1 != start ||
  1295. !state->state & EXTENT_WRITEBACK)) {
  1296. state = NULL;
  1297. }
  1298. spin_unlock_irqrestore(&tree->lock, flags);
  1299. next_io:
  1300. if (whole_page)
  1301. end_page_writeback(page);
  1302. else
  1303. check_page_writeback(tree, page);
  1304. } while (bvec >= bio->bi_io_vec);
  1305. bio_put(bio);
  1306. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1307. return 0;
  1308. #endif
  1309. }
  1310. /*
  1311. * after a readpage IO is done, we need to:
  1312. * clear the uptodate bits on error
  1313. * set the uptodate bits if things worked
  1314. * set the page up to date if all extents in the tree are uptodate
  1315. * clear the lock bit in the extent tree
  1316. * unlock the page if there are no other extents locked for it
  1317. *
  1318. * Scheduling is not allowed, so the extent state tree is expected
  1319. * to have one and only one object corresponding to this IO.
  1320. */
  1321. #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
  1322. static void end_bio_extent_readpage(struct bio *bio, int err)
  1323. #else
  1324. static int end_bio_extent_readpage(struct bio *bio,
  1325. unsigned int bytes_done, int err)
  1326. #endif
  1327. {
  1328. int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  1329. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1330. struct extent_state *state = bio->bi_private;
  1331. struct extent_io_tree *tree = state->tree;
  1332. struct rb_node *node;
  1333. u64 start;
  1334. u64 end;
  1335. u64 cur;
  1336. unsigned long flags;
  1337. int whole_page;
  1338. int ret;
  1339. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1340. if (bio->bi_size)
  1341. return 1;
  1342. #endif
  1343. do {
  1344. struct page *page = bvec->bv_page;
  1345. start = ((u64)page->index << PAGE_CACHE_SHIFT) +
  1346. bvec->bv_offset;
  1347. end = start + bvec->bv_len - 1;
  1348. if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
  1349. whole_page = 1;
  1350. else
  1351. whole_page = 0;
  1352. if (--bvec >= bio->bi_io_vec)
  1353. prefetchw(&bvec->bv_page->flags);
  1354. if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
  1355. ret = tree->ops->readpage_end_io_hook(page, start, end,
  1356. state);
  1357. if (ret)
  1358. uptodate = 0;
  1359. }
  1360. spin_lock_irqsave(&tree->lock, flags);
  1361. if (!state || state->end != end) {
  1362. state = NULL;
  1363. node = __etree_search(tree, start, NULL, NULL);
  1364. if (node) {
  1365. state = rb_entry(node, struct extent_state,
  1366. rb_node);
  1367. if (state->end != end ||
  1368. !(state->state & EXTENT_LOCKED))
  1369. state = NULL;
  1370. }
  1371. if (!state) {
  1372. spin_unlock_irqrestore(&tree->lock, flags);
  1373. set_extent_uptodate(tree, start, end,
  1374. GFP_ATOMIC);
  1375. unlock_extent(tree, start, end, GFP_ATOMIC);
  1376. goto next_io;
  1377. }
  1378. }
  1379. cur = end;
  1380. while(1) {
  1381. struct extent_state *clear = state;
  1382. cur = state->start;
  1383. node = rb_prev(&state->rb_node);
  1384. if (node) {
  1385. state = rb_entry(node,
  1386. struct extent_state,
  1387. rb_node);
  1388. } else {
  1389. state = NULL;
  1390. }
  1391. set_state_cb(tree, clear, EXTENT_UPTODATE);
  1392. clear->state |= EXTENT_UPTODATE;
  1393. clear_state_bit(tree, clear, EXTENT_LOCKED,
  1394. 1, 0);
  1395. if (cur == start)
  1396. break;
  1397. if (cur < start) {
  1398. WARN_ON(1);
  1399. break;
  1400. }
  1401. if (!node)
  1402. break;
  1403. }
  1404. /* before releasing the lock, make sure the next state
  1405. * variable has the expected bits set and corresponds
  1406. * to the correct offsets in the file
  1407. */
  1408. if (state && (state->end + 1 != start ||
  1409. !state->state & EXTENT_WRITEBACK)) {
  1410. state = NULL;
  1411. }
  1412. spin_unlock_irqrestore(&tree->lock, flags);
  1413. next_io:
  1414. if (whole_page) {
  1415. if (uptodate) {
  1416. SetPageUptodate(page);
  1417. } else {
  1418. ClearPageUptodate(page);
  1419. SetPageError(page);
  1420. }
  1421. unlock_page(page);
  1422. } else {
  1423. if (uptodate) {
  1424. check_page_uptodate(tree, page);
  1425. } else {
  1426. ClearPageUptodate(page);
  1427. SetPageError(page);
  1428. }
  1429. check_page_locked(tree, page);
  1430. }
  1431. } while (bvec >= bio->bi_io_vec);
  1432. bio_put(bio);
  1433. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1434. return 0;
  1435. #endif
  1436. }
  1437. /*
  1438. * IO done from prepare_write is pretty simple, we just unlock
  1439. * the structs in the extent tree when done, and set the uptodate bits
  1440. * as appropriate.
  1441. */
  1442. #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
  1443. static void end_bio_extent_preparewrite(struct bio *bio, int err)
  1444. #else
  1445. static int end_bio_extent_preparewrite(struct bio *bio,
  1446. unsigned int bytes_done, int err)
  1447. #endif
  1448. {
  1449. const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  1450. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1451. struct extent_state *state = bio->bi_private;
  1452. struct extent_io_tree *tree = state->tree;
  1453. u64 start;
  1454. u64 end;
  1455. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1456. if (bio->bi_size)
  1457. return 1;
  1458. #endif
  1459. do {
  1460. struct page *page = bvec->bv_page;
  1461. start = ((u64)page->index << PAGE_CACHE_SHIFT) +
  1462. bvec->bv_offset;
  1463. end = start + bvec->bv_len - 1;
  1464. if (--bvec >= bio->bi_io_vec)
  1465. prefetchw(&bvec->bv_page->flags);
  1466. if (uptodate) {
  1467. set_extent_uptodate(tree, start, end, GFP_ATOMIC);
  1468. } else {
  1469. ClearPageUptodate(page);
  1470. SetPageError(page);
  1471. }
  1472. unlock_extent(tree, start, end, GFP_ATOMIC);
  1473. } while (bvec >= bio->bi_io_vec);
  1474. bio_put(bio);
  1475. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
  1476. return 0;
  1477. #endif
  1478. }
  1479. static struct bio *
  1480. extent_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
  1481. gfp_t gfp_flags)
  1482. {
  1483. struct bio *bio;
  1484. bio = bio_alloc(gfp_flags, nr_vecs);
  1485. if (bio == NULL && (current->flags & PF_MEMALLOC)) {
  1486. while (!bio && (nr_vecs /= 2))
  1487. bio = bio_alloc(gfp_flags, nr_vecs);
  1488. }
  1489. if (bio) {
  1490. bio->bi_bdev = bdev;
  1491. bio->bi_sector = first_sector;
  1492. }
  1493. return bio;
  1494. }
  1495. static int submit_one_bio(int rw, struct bio *bio)
  1496. {
  1497. u64 maxsector;
  1498. int ret = 0;
  1499. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  1500. struct page *page = bvec->bv_page;
  1501. struct extent_io_tree *tree = bio->bi_private;
  1502. struct rb_node *node;
  1503. struct extent_state *state;
  1504. u64 start;
  1505. u64 end;
  1506. start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset;
  1507. end = start + bvec->bv_len - 1;
  1508. spin_lock_irq(&tree->lock);
  1509. node = __etree_search(tree, start, NULL, NULL);
  1510. BUG_ON(!node);
  1511. state = rb_entry(node, struct extent_state, rb_node);
  1512. while(state->end < end) {
  1513. node = rb_next(node);
  1514. state = rb_entry(node, struct extent_state, rb_node);
  1515. }
  1516. BUG_ON(state->end != end);
  1517. spin_unlock_irq(&tree->lock);
  1518. bio->bi_private = state;
  1519. bio_get(bio);
  1520. maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
  1521. if (maxsector < bio->bi_sector) {
  1522. printk("sector too large max %Lu got %llu\n", maxsector,
  1523. (unsigned long long)bio->bi_sector);
  1524. WARN_ON(1);
  1525. }
  1526. submit_bio(rw, bio);
  1527. if (bio_flagged(bio, BIO_EOPNOTSUPP))
  1528. ret = -EOPNOTSUPP;
  1529. bio_put(bio);
  1530. return ret;
  1531. }
  1532. static int submit_extent_page(int rw, struct extent_io_tree *tree,
  1533. struct page *page, sector_t sector,
  1534. size_t size, unsigned long offset,
  1535. struct block_device *bdev,
  1536. struct bio **bio_ret,
  1537. unsigned long max_pages,
  1538. bio_end_io_t end_io_func)
  1539. {
  1540. int ret = 0;
  1541. struct bio *bio;
  1542. int nr;
  1543. if (bio_ret && *bio_ret) {
  1544. bio = *bio_ret;
  1545. if (bio->bi_sector + (bio->bi_size >> 9) != sector ||
  1546. bio_add_page(bio, page, size, offset) < size) {
  1547. ret = submit_one_bio(rw, bio);
  1548. bio = NULL;
  1549. } else {
  1550. return 0;
  1551. }
  1552. }
  1553. nr = min_t(int, max_pages, bio_get_nr_vecs(bdev));
  1554. bio = extent_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
  1555. if (!bio) {
  1556. printk("failed to allocate bio nr %d\n", nr);
  1557. }
  1558. bio_add_page(bio, page, size, offset);
  1559. bio->bi_end_io = end_io_func;
  1560. bio->bi_private = tree;
  1561. if (bio_ret) {
  1562. *bio_ret = bio;
  1563. } else {
  1564. ret = submit_one_bio(rw, bio);
  1565. }
  1566. return ret;
  1567. }
  1568. void set_page_extent_mapped(struct page *page)
  1569. {
  1570. if (!PagePrivate(page)) {
  1571. SetPagePrivate(page);
  1572. WARN_ON(!page->mapping->a_ops->invalidatepage);
  1573. set_page_private(page, EXTENT_PAGE_PRIVATE);
  1574. page_cache_get(page);
  1575. }
  1576. }
  1577. void set_page_extent_head(struct page *page, unsigned long len)
  1578. {
  1579. set_page_private(page, EXTENT_PAGE_PRIVATE_FIRST_PAGE | len << 2);
  1580. }
  1581. /*
  1582. * basic readpage implementation. Locked extent state structs are inserted
  1583. * into the tree that are removed when the IO is done (by the end_io
  1584. * handlers)
  1585. */
  1586. static int __extent_read_full_page(struct extent_io_tree *tree,
  1587. struct page *page,
  1588. get_extent_t *get_extent,
  1589. struct bio **bio)
  1590. {
  1591. struct inode *inode = page->mapping->host;
  1592. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1593. u64 page_end = start + PAGE_CACHE_SIZE - 1;
  1594. u64 end;
  1595. u64 cur = start;
  1596. u64 extent_offset;
  1597. u64 last_byte = i_size_read(inode);
  1598. u64 block_start;
  1599. u64 cur_end;
  1600. sector_t sector;
  1601. struct extent_map *em;
  1602. struct block_device *bdev;
  1603. int ret;
  1604. int nr = 0;
  1605. size_t page_offset = 0;
  1606. size_t iosize;
  1607. size_t blocksize = inode->i_sb->s_blocksize;
  1608. set_page_extent_mapped(page);
  1609. end = page_end;
  1610. lock_extent(tree, start, end, GFP_NOFS);
  1611. while (cur <= end) {
  1612. if (cur >= last_byte) {
  1613. char *userpage;
  1614. iosize = PAGE_CACHE_SIZE - page_offset;
  1615. userpage = kmap_atomic(page, KM_USER0);
  1616. memset(userpage + page_offset, 0, iosize);
  1617. flush_dcache_page(page);
  1618. kunmap_atomic(userpage, KM_USER0);
  1619. set_extent_uptodate(tree, cur, cur + iosize - 1,
  1620. GFP_NOFS);
  1621. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1622. break;
  1623. }
  1624. em = get_extent(inode, page, page_offset, cur,
  1625. end - cur + 1, 0);
  1626. if (IS_ERR(em) || !em) {
  1627. SetPageError(page);
  1628. unlock_extent(tree, cur, end, GFP_NOFS);
  1629. break;
  1630. }
  1631. extent_offset = cur - em->start;
  1632. BUG_ON(extent_map_end(em) <= cur);
  1633. BUG_ON(end < cur);
  1634. iosize = min(extent_map_end(em) - cur, end - cur + 1);
  1635. cur_end = min(extent_map_end(em) - 1, end);
  1636. iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
  1637. sector = (em->block_start + extent_offset) >> 9;
  1638. bdev = em->bdev;
  1639. block_start = em->block_start;
  1640. free_extent_map(em);
  1641. em = NULL;
  1642. /* we've found a hole, just zero and go on */
  1643. if (block_start == EXTENT_MAP_HOLE) {
  1644. char *userpage;
  1645. userpage = kmap_atomic(page, KM_USER0);
  1646. memset(userpage + page_offset, 0, iosize);
  1647. flush_dcache_page(page);
  1648. kunmap_atomic(userpage, KM_USER0);
  1649. set_extent_uptodate(tree, cur, cur + iosize - 1,
  1650. GFP_NOFS);
  1651. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1652. cur = cur + iosize;
  1653. page_offset += iosize;
  1654. continue;
  1655. }
  1656. /* the get_extent function already copied into the page */
  1657. if (test_range_bit(tree, cur, cur_end, EXTENT_UPTODATE, 1)) {
  1658. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1659. cur = cur + iosize;
  1660. page_offset += iosize;
  1661. continue;
  1662. }
  1663. /* we have an inline extent but it didn't get marked up
  1664. * to date. Error out
  1665. */
  1666. if (block_start == EXTENT_MAP_INLINE) {
  1667. SetPageError(page);
  1668. unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
  1669. cur = cur + iosize;
  1670. page_offset += iosize;
  1671. continue;
  1672. }
  1673. ret = 0;
  1674. if (tree->ops && tree->ops->readpage_io_hook) {
  1675. ret = tree->ops->readpage_io_hook(page, cur,
  1676. cur + iosize - 1);
  1677. }
  1678. if (!ret) {
  1679. unsigned long nr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
  1680. nr -= page->index;
  1681. ret = submit_extent_page(READ, tree, page,
  1682. sector, iosize, page_offset,
  1683. bdev, bio, nr,
  1684. end_bio_extent_readpage);
  1685. }
  1686. if (ret)
  1687. SetPageError(page);
  1688. cur = cur + iosize;
  1689. page_offset += iosize;
  1690. nr++;
  1691. }
  1692. if (!nr) {
  1693. if (!PageError(page))
  1694. SetPageUptodate(page);
  1695. unlock_page(page);
  1696. }
  1697. return 0;
  1698. }
  1699. int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
  1700. get_extent_t *get_extent)
  1701. {
  1702. struct bio *bio = NULL;
  1703. int ret;
  1704. ret = __extent_read_full_page(tree, page, get_extent, &bio);
  1705. if (bio)
  1706. submit_one_bio(READ, bio);
  1707. return ret;
  1708. }
  1709. EXPORT_SYMBOL(extent_read_full_page);
  1710. /*
  1711. * the writepage semantics are similar to regular writepage. extent
  1712. * records are inserted to lock ranges in the tree, and as dirty areas
  1713. * are found, they are marked writeback. Then the lock bits are removed
  1714. * and the end_io handler clears the writeback ranges
  1715. */
  1716. static int __extent_writepage(struct page *page, struct writeback_control *wbc,
  1717. void *data)
  1718. {
  1719. struct inode *inode = page->mapping->host;
  1720. struct extent_page_data *epd = data;
  1721. struct extent_io_tree *tree = epd->tree;
  1722. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  1723. u64 delalloc_start;
  1724. u64 page_end = start + PAGE_CACHE_SIZE - 1;
  1725. u64 end;
  1726. u64 cur = start;
  1727. u64 extent_offset;
  1728. u64 last_byte = i_size_read(inode);
  1729. u64 block_start;
  1730. u64 iosize;
  1731. sector_t sector;
  1732. struct extent_map *em;
  1733. struct block_device *bdev;
  1734. int ret;
  1735. int nr = 0;
  1736. size_t page_offset = 0;
  1737. size_t blocksize;
  1738. loff_t i_size = i_size_read(inode);
  1739. unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
  1740. u64 nr_delalloc;
  1741. u64 delalloc_end;
  1742. WARN_ON(!PageLocked(page));
  1743. if (page->index > end_index) {
  1744. clear_extent_dirty(tree, start, page_end, GFP_NOFS);
  1745. unlock_page(page);
  1746. return 0;
  1747. }
  1748. if (page->index == end_index) {
  1749. char *userpage;
  1750. size_t offset = i_size & (PAGE_CACHE_SIZE - 1);
  1751. userpage = kmap_atomic(page, KM_USER0);
  1752. memset(userpage + offset, 0, PAGE_CACHE_SIZE - offset);
  1753. flush_dcache_page(page);
  1754. kunmap_atomic(userpage, KM_USER0);
  1755. }
  1756. set_page_extent_mapped(page);
  1757. delalloc_start = start;
  1758. delalloc_end = 0;
  1759. while(delalloc_end < page_end) {
  1760. nr_delalloc = find_lock_delalloc_range(tree, &delalloc_start,
  1761. &delalloc_end,
  1762. 128 * 1024 * 1024);
  1763. if (nr_delalloc == 0) {
  1764. delalloc_start = delalloc_end + 1;
  1765. continue;
  1766. }
  1767. tree->ops->fill_delalloc(inode, delalloc_start,
  1768. delalloc_end);
  1769. clear_extent_bit(tree, delalloc_start,
  1770. delalloc_end,
  1771. EXTENT_LOCKED | EXTENT_DELALLOC,
  1772. 1, 0, GFP_NOFS);
  1773. delalloc_start = delalloc_end + 1;
  1774. }
  1775. lock_extent(tree, start, page_end, GFP_NOFS);
  1776. end = page_end;
  1777. if (test_range_bit(tree, start, page_end, EXTENT_DELALLOC, 0)) {
  1778. printk("found delalloc bits after lock_extent\n");
  1779. }
  1780. if (last_byte <= start) {
  1781. clear_extent_dirty(tree, start, page_end, GFP_NOFS);
  1782. goto done;
  1783. }
  1784. set_extent_uptodate(tree, start, page_end, GFP_NOFS);
  1785. blocksize = inode->i_sb->s_blocksize;
  1786. while (cur <= end) {
  1787. if (cur >= last_byte) {
  1788. clear_extent_dirty(tree, cur, page_end, GFP_NOFS);
  1789. break;
  1790. }
  1791. em = epd->get_extent(inode, page, page_offset, cur,
  1792. end - cur + 1, 1);
  1793. if (IS_ERR(em) || !em) {
  1794. SetPageError(page);
  1795. break;
  1796. }
  1797. extent_offset = cur - em->start;
  1798. BUG_ON(extent_map_end(em) <= cur);
  1799. BUG_ON(end < cur);
  1800. iosize = min(extent_map_end(em) - cur, end - cur + 1);
  1801. iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
  1802. sector = (em->block_start + extent_offset) >> 9;
  1803. bdev = em->bdev;
  1804. block_start = em->block_start;
  1805. free_extent_map(em);
  1806. em = NULL;
  1807. if (block_start == EXTENT_MAP_HOLE ||
  1808. block_start == EXTENT_MAP_INLINE) {
  1809. clear_extent_dirty(tree, cur,
  1810. cur + iosize - 1, GFP_NOFS);
  1811. cur = cur + iosize;
  1812. page_offset += iosize;
  1813. continue;
  1814. }
  1815. /* leave this out until we have a page_mkwrite call */
  1816. if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
  1817. EXTENT_DIRTY, 0)) {
  1818. cur = cur + iosize;
  1819. page_offset += iosize;
  1820. continue;
  1821. }
  1822. clear_extent_dirty(tree, cur, cur + iosize - 1, GFP_NOFS);
  1823. if (tree->ops && tree->ops->writepage_io_hook) {
  1824. ret = tree->ops->writepage_io_hook(page, cur,
  1825. cur + iosize - 1);
  1826. } else {
  1827. ret = 0;
  1828. }
  1829. if (ret)
  1830. SetPageError(page);
  1831. else {
  1832. unsigned long max_nr = end_index + 1;
  1833. set_range_writeback(tree, cur, cur + iosize - 1);
  1834. if (!PageWriteback(page)) {
  1835. printk("warning page %lu not writeback, "
  1836. "cur %llu end %llu\n", page->index,
  1837. (unsigned long long)cur,
  1838. (unsigned long long)end);
  1839. }
  1840. ret = submit_extent_page(WRITE, tree, page, sector,
  1841. iosize, page_offset, bdev,
  1842. &epd->bio, max_nr,
  1843. end_bio_extent_writepage);
  1844. if (ret)
  1845. SetPageError(page);
  1846. }
  1847. cur = cur + iosize;
  1848. page_offset += iosize;
  1849. nr++;
  1850. }
  1851. done:
  1852. if (nr == 0) {
  1853. /* make sure the mapping tag for page dirty gets cleared */
  1854. set_page_writeback(page);
  1855. end_page_writeback(page);
  1856. }
  1857. unlock_extent(tree, start, page_end, GFP_NOFS);
  1858. unlock_page(page);
  1859. return 0;
  1860. }
  1861. #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
  1862. /* Taken directly from 2.6.23 for 2.6.18 back port */
  1863. typedef int (*writepage_t)(struct page *page, struct writeback_control *wbc,
  1864. void *data);
  1865. /**
  1866. * write_cache_pages - walk the list of dirty pages of the given address space
  1867. * and write all of them.
  1868. * @mapping: address space structure to write
  1869. * @wbc: subtract the number of written pages from *@wbc->nr_to_write
  1870. * @writepage: function called for each page
  1871. * @data: data passed to writepage function
  1872. *
  1873. * If a page is already under I/O, write_cache_pages() skips it, even
  1874. * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
  1875. * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
  1876. * and msync() need to guarantee that all the data which was dirty at the time
  1877. * the call was made get new I/O started against them. If wbc->sync_mode is
  1878. * WB_SYNC_ALL then we were called for data integrity and we must wait for
  1879. * existing IO to complete.
  1880. */
  1881. static int write_cache_pages(struct address_space *mapping,
  1882. struct writeback_control *wbc, writepage_t writepage,
  1883. void *data)
  1884. {
  1885. struct backing_dev_info *bdi = mapping->backing_dev_info;
  1886. int ret = 0;
  1887. int done = 0;
  1888. struct pagevec pvec;
  1889. int nr_pages;
  1890. pgoff_t index;
  1891. pgoff_t end; /* Inclusive */
  1892. int scanned = 0;
  1893. int range_whole = 0;
  1894. if (wbc->nonblocking && bdi_write_congested(bdi)) {
  1895. wbc->encountered_congestion = 1;
  1896. return 0;
  1897. }
  1898. pagevec_init(&pvec, 0);
  1899. if (wbc->range_cyclic) {
  1900. index = mapping->writeback_index; /* Start from prev offset */
  1901. end = -1;
  1902. } else {
  1903. index = wbc->range_start >> PAGE_CACHE_SHIFT;
  1904. end = wbc->range_end >> PAGE_CACHE_SHIFT;
  1905. if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
  1906. range_whole = 1;
  1907. scanned = 1;
  1908. }
  1909. retry:
  1910. while (!done && (index <= end) &&
  1911. (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
  1912. PAGECACHE_TAG_DIRTY,
  1913. min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
  1914. unsigned i;
  1915. scanned = 1;
  1916. for (i = 0; i < nr_pages; i++) {
  1917. struct page *page = pvec.pages[i];
  1918. /*
  1919. * At this point we hold neither mapping->tree_lock nor
  1920. * lock on the page itself: the page may be truncated or
  1921. * invalidated (changing page->mapping to NULL), or even
  1922. * swizzled back from swapper_space to tmpfs file
  1923. * mapping
  1924. */
  1925. lock_page(page);
  1926. if (unlikely(page->mapping != mapping)) {
  1927. unlock_page(page);
  1928. continue;
  1929. }
  1930. if (!wbc->range_cyclic && page->index > end) {
  1931. done = 1;
  1932. unlock_page(page);
  1933. continue;
  1934. }
  1935. if (wbc->sync_mode != WB_SYNC_NONE)
  1936. wait_on_page_writeback(page);
  1937. if (PageWriteback(page) ||
  1938. !clear_page_dirty_for_io(page)) {
  1939. unlock_page(page);
  1940. continue;
  1941. }
  1942. ret = (*writepage)(page, wbc, data);
  1943. if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
  1944. unlock_page(page);
  1945. ret = 0;
  1946. }
  1947. if (ret || (--(wbc->nr_to_write) <= 0))
  1948. done = 1;
  1949. if (wbc->nonblocking && bdi_write_congested(bdi)) {
  1950. wbc->encountered_congestion = 1;
  1951. done = 1;
  1952. }
  1953. }
  1954. pagevec_release(&pvec);
  1955. cond_resched();
  1956. }
  1957. if (!scanned && !done) {
  1958. /*
  1959. * We hit the last page and there is more work to be done: wrap
  1960. * back to the start of the file
  1961. */
  1962. scanned = 1;
  1963. index = 0;
  1964. goto retry;
  1965. }
  1966. if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
  1967. mapping->writeback_index = index;
  1968. return ret;
  1969. }
  1970. #endif
  1971. int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
  1972. get_extent_t *get_extent,
  1973. struct writeback_control *wbc)
  1974. {
  1975. int ret;
  1976. struct address_space *mapping = page->mapping;
  1977. struct extent_page_data epd = {
  1978. .bio = NULL,
  1979. .tree = tree,
  1980. .get_extent = get_extent,
  1981. };
  1982. struct writeback_control wbc_writepages = {
  1983. .bdi = wbc->bdi,
  1984. .sync_mode = WB_SYNC_NONE,
  1985. .older_than_this = NULL,
  1986. .nr_to_write = 64,
  1987. .range_start = page_offset(page) + PAGE_CACHE_SIZE,
  1988. .range_end = (loff_t)-1,
  1989. };
  1990. ret = __extent_writepage(page, wbc, &epd);
  1991. write_cache_pages(mapping, &wbc_writepages, __extent_writepage, &epd);
  1992. if (epd.bio) {
  1993. submit_one_bio(WRITE, epd.bio);
  1994. }
  1995. return ret;
  1996. }
  1997. EXPORT_SYMBOL(extent_write_full_page);
  1998. int extent_writepages(struct extent_io_tree *tree,
  1999. struct address_space *mapping,
  2000. get_extent_t *get_extent,
  2001. struct writeback_control *wbc)
  2002. {
  2003. int ret = 0;
  2004. struct extent_page_data epd = {
  2005. .bio = NULL,
  2006. .tree = tree,
  2007. .get_extent = get_extent,
  2008. };
  2009. ret = write_cache_pages(mapping, wbc, __extent_writepage, &epd);
  2010. if (epd.bio) {
  2011. submit_one_bio(WRITE, epd.bio);
  2012. }
  2013. return ret;
  2014. }
  2015. EXPORT_SYMBOL(extent_writepages);
  2016. int extent_readpages(struct extent_io_tree *tree,
  2017. struct address_space *mapping,
  2018. struct list_head *pages, unsigned nr_pages,
  2019. get_extent_t get_extent)
  2020. {
  2021. struct bio *bio = NULL;
  2022. unsigned page_idx;
  2023. struct pagevec pvec;
  2024. pagevec_init(&pvec, 0);
  2025. for (page_idx = 0; page_idx < nr_pages; page_idx++) {
  2026. struct page *page = list_entry(pages->prev, struct page, lru);
  2027. prefetchw(&page->flags);
  2028. list_del(&page->lru);
  2029. /*
  2030. * what we want to do here is call add_to_page_cache_lru,
  2031. * but that isn't exported, so we reproduce it here
  2032. */
  2033. if (!add_to_page_cache(page, mapping,
  2034. page->index, GFP_KERNEL)) {
  2035. /* open coding of lru_cache_add, also not exported */
  2036. page_cache_get(page);
  2037. if (!pagevec_add(&pvec, page))
  2038. __pagevec_lru_add(&pvec);
  2039. __extent_read_full_page(tree, page, get_extent, &bio);
  2040. }
  2041. page_cache_release(page);
  2042. }
  2043. if (pagevec_count(&pvec))
  2044. __pagevec_lru_add(&pvec);
  2045. BUG_ON(!list_empty(pages));
  2046. if (bio)
  2047. submit_one_bio(READ, bio);
  2048. return 0;
  2049. }
  2050. EXPORT_SYMBOL(extent_readpages);
  2051. /*
  2052. * basic invalidatepage code, this waits on any locked or writeback
  2053. * ranges corresponding to the page, and then deletes any extent state
  2054. * records from the tree
  2055. */
  2056. int extent_invalidatepage(struct extent_io_tree *tree,
  2057. struct page *page, unsigned long offset)
  2058. {
  2059. u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
  2060. u64 end = start + PAGE_CACHE_SIZE - 1;
  2061. size_t blocksize = page->mapping->host->i_sb->s_blocksize;
  2062. start += (offset + blocksize -1) & ~(blocksize - 1);
  2063. if (start > end)
  2064. return 0;
  2065. lock_extent(tree, start, end, GFP_NOFS);
  2066. wait_on_extent_writeback(tree, start, end);
  2067. clear_extent_bit(tree, start, end,
  2068. EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC,
  2069. 1, 1, GFP_NOFS);
  2070. return 0;
  2071. }
  2072. EXPORT_SYMBOL(extent_invalidatepage);
  2073. /*
  2074. * simple commit_write call, set_range_dirty is used to mark both
  2075. * the pages and the extent records as dirty
  2076. */
  2077. int extent_commit_write(struct extent_io_tree *tree,
  2078. struct inode *inode, struct page *page,
  2079. unsigned from, unsigned to)
  2080. {
  2081. loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
  2082. set_page_extent_mapped(page);
  2083. set_page_dirty(page);
  2084. if (pos > inode->i_size) {
  2085. i_size_write(inode, pos);
  2086. mark_inode_dirty(inode);
  2087. }
  2088. return 0;
  2089. }
  2090. EXPORT_SYMBOL(extent_commit_write);
  2091. int extent_prepare_write(struct extent_io_tree *tree,
  2092. struct inode *inode, struct page *page,
  2093. unsigned from, unsigned to, get_extent_t *get_extent)
  2094. {
  2095. u64 page_start = (u64)page->index << PAGE_CACHE_SHIFT;
  2096. u64 page_end = page_start + PAGE_CACHE_SIZE - 1;
  2097. u64 block_start;
  2098. u64 orig_block_start;
  2099. u64 block_end;
  2100. u64 cur_end;
  2101. struct extent_map *em;
  2102. unsigned blocksize = 1 << inode->i_blkbits;
  2103. size_t page_offset = 0;
  2104. size_t block_off_start;
  2105. size_t block_off_end;
  2106. int err = 0;
  2107. int iocount = 0;
  2108. int ret = 0;
  2109. int isnew;
  2110. set_page_extent_mapped(page);
  2111. block_start = (page_start + from) & ~((u64)blocksize - 1);
  2112. block_end = (page_start + to - 1) | (blocksize - 1);
  2113. orig_block_start = block_start;
  2114. lock_extent(tree, page_start, page_end, GFP_NOFS);
  2115. while(block_start <= block_end) {
  2116. em = get_extent(inode, page, page_offset, block_start,
  2117. block_end - block_start + 1, 1);
  2118. if (IS_ERR(em) || !em) {
  2119. goto err;
  2120. }
  2121. cur_end = min(block_end, extent_map_end(em) - 1);
  2122. block_off_start = block_start & (PAGE_CACHE_SIZE - 1);
  2123. block_off_end = block_off_start + blocksize;
  2124. isnew = clear_extent_new(tree, block_start, cur_end, GFP_NOFS);
  2125. if (!PageUptodate(page) && isnew &&
  2126. (block_off_end > to || block_off_start < from)) {
  2127. void *kaddr;
  2128. kaddr = kmap_atomic(page, KM_USER0);
  2129. if (block_off_end > to)
  2130. memset(kaddr + to, 0, block_off_end - to);
  2131. if (block_off_start < from)
  2132. memset(kaddr + block_off_start, 0,
  2133. from - block_off_start);
  2134. flush_dcache_page(page);
  2135. kunmap_atomic(kaddr, KM_USER0);
  2136. }
  2137. if ((em->block_start != EXTENT_MAP_HOLE &&
  2138. em->block_start != EXTENT_MAP_INLINE) &&
  2139. !isnew && !PageUptodate(page) &&
  2140. (block_off_end > to || block_off_start < from) &&
  2141. !test_range_bit(tree, block_start, cur_end,
  2142. EXTENT_UPTODATE, 1)) {
  2143. u64 sector;
  2144. u64 extent_offset = block_start - em->start;
  2145. size_t iosize;
  2146. sector = (em->block_start + extent_offset) >> 9;
  2147. iosize = (cur_end - block_start + blocksize) &
  2148. ~((u64)blocksize - 1);
  2149. /*
  2150. * we've already got the extent locked, but we
  2151. * need to split the state such that our end_bio
  2152. * handler can clear the lock.
  2153. */
  2154. set_extent_bit(tree, block_start,
  2155. block_start + iosize - 1,
  2156. EXTENT_LOCKED, 0, NULL, GFP_NOFS);
  2157. ret = submit_extent_page(READ, tree, page,
  2158. sector, iosize, page_offset, em->bdev,
  2159. NULL, 1,
  2160. end_bio_extent_preparewrite);
  2161. iocount++;
  2162. block_start = block_start + iosize;
  2163. } else {
  2164. set_extent_uptodate(tree, block_start, cur_end,
  2165. GFP_NOFS);
  2166. unlock_extent(tree, block_start, cur_end, GFP_NOFS);
  2167. block_start = cur_end + 1;
  2168. }
  2169. page_offset = block_start & (PAGE_CACHE_SIZE - 1);
  2170. free_extent_map(em);
  2171. }
  2172. if (iocount) {
  2173. wait_extent_bit(tree, orig_block_start,
  2174. block_end, EXTENT_LOCKED);
  2175. }
  2176. check_page_uptodate(tree, page);
  2177. err:
  2178. /* FIXME, zero out newly allocated blocks on error */
  2179. return err;
  2180. }
  2181. EXPORT_SYMBOL(extent_prepare_write);
  2182. /*
  2183. * a helper for releasepage. As long as there are no locked extents
  2184. * in the range corresponding to the page, both state records and extent
  2185. * map records are removed
  2186. */
  2187. int try_release_extent_mapping(struct extent_map_tree *map,
  2188. struct extent_io_tree *tree, struct page *page,
  2189. gfp_t mask)
  2190. {
  2191. struct extent_map *em;
  2192. u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
  2193. u64 end = start + PAGE_CACHE_SIZE - 1;
  2194. u64 orig_start = start;
  2195. int ret = 1;
  2196. if ((mask & __GFP_WAIT) &&
  2197. page->mapping->host->i_size > 16 * 1024 * 1024) {
  2198. while (start <= end) {
  2199. spin_lock(&map->lock);
  2200. em = lookup_extent_mapping(map, start, end);
  2201. if (!em || IS_ERR(em)) {
  2202. spin_unlock(&map->lock);
  2203. break;
  2204. }
  2205. if (em->start != start) {
  2206. spin_unlock(&map->lock);
  2207. free_extent_map(em);
  2208. break;
  2209. }
  2210. if (!test_range_bit(tree, em->start,
  2211. extent_map_end(em) - 1,
  2212. EXTENT_LOCKED, 0)) {
  2213. remove_extent_mapping(map, em);
  2214. /* once for the rb tree */
  2215. free_extent_map(em);
  2216. }
  2217. start = extent_map_end(em);
  2218. spin_unlock(&map->lock);
  2219. /* once for us */
  2220. free_extent_map(em);
  2221. }
  2222. }
  2223. if (test_range_bit(tree, orig_start, end, EXTENT_IOBITS, 0))
  2224. ret = 0;
  2225. else {
  2226. if ((mask & GFP_NOFS) == GFP_NOFS)
  2227. mask = GFP_NOFS;
  2228. clear_extent_bit(tree, orig_start, end, EXTENT_UPTODATE,
  2229. 1, 1, mask);
  2230. }
  2231. return ret;
  2232. }
  2233. EXPORT_SYMBOL(try_release_extent_mapping);
  2234. sector_t extent_bmap(struct address_space *mapping, sector_t iblock,
  2235. get_extent_t *get_extent)
  2236. {
  2237. struct inode *inode = mapping->host;
  2238. u64 start = iblock << inode->i_blkbits;
  2239. sector_t sector = 0;
  2240. struct extent_map *em;
  2241. em = get_extent(inode, NULL, 0, start, (1 << inode->i_blkbits), 0);
  2242. if (!em || IS_ERR(em))
  2243. return 0;
  2244. if (em->block_start == EXTENT_MAP_INLINE ||
  2245. em->block_start == EXTENT_MAP_HOLE)
  2246. goto out;
  2247. sector = (em->block_start + start - em->start) >> inode->i_blkbits;
  2248. out:
  2249. free_extent_map(em);
  2250. return sector;
  2251. }
  2252. static int add_lru(struct extent_io_tree *tree, struct extent_buffer *eb)
  2253. {
  2254. if (list_empty(&eb->lru)) {
  2255. extent_buffer_get(eb);
  2256. list_add(&eb->lru, &tree->buffer_lru);
  2257. tree->lru_size++;
  2258. if (tree->lru_size >= BUFFER_LRU_MAX) {
  2259. struct extent_buffer *rm;
  2260. rm = list_entry(tree->buffer_lru.prev,
  2261. struct extent_buffer, lru);
  2262. tree->lru_size--;
  2263. list_del_init(&rm->lru);
  2264. free_extent_buffer(rm);
  2265. }
  2266. } else
  2267. list_move(&eb->lru, &tree->buffer_lru);
  2268. return 0;
  2269. }
  2270. static struct extent_buffer *find_lru(struct extent_io_tree *tree,
  2271. u64 start, unsigned long len)
  2272. {
  2273. struct list_head *lru = &tree->buffer_lru;
  2274. struct list_head *cur = lru->next;
  2275. struct extent_buffer *eb;
  2276. if (list_empty(lru))
  2277. return NULL;
  2278. do {
  2279. eb = list_entry(cur, struct extent_buffer, lru);
  2280. if (eb->start == start && eb->len == len) {
  2281. extent_buffer_get(eb);
  2282. return eb;
  2283. }
  2284. cur = cur->next;
  2285. } while (cur != lru);
  2286. return NULL;
  2287. }
  2288. static inline unsigned long num_extent_pages(u64 start, u64 len)
  2289. {
  2290. return ((start + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) -
  2291. (start >> PAGE_CACHE_SHIFT);
  2292. }
  2293. static inline struct page *extent_buffer_page(struct extent_buffer *eb,
  2294. unsigned long i)
  2295. {
  2296. struct page *p;
  2297. struct address_space *mapping;
  2298. if (i == 0)
  2299. return eb->first_page;
  2300. i += eb->start >> PAGE_CACHE_SHIFT;
  2301. mapping = eb->first_page->mapping;
  2302. read_lock_irq(&mapping->tree_lock);
  2303. p = radix_tree_lookup(&mapping->page_tree, i);
  2304. read_unlock_irq(&mapping->tree_lock);
  2305. return p;
  2306. }
  2307. static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
  2308. u64 start,
  2309. unsigned long len,
  2310. gfp_t mask)
  2311. {
  2312. struct extent_buffer *eb = NULL;
  2313. spin_lock(&tree->lru_lock);
  2314. eb = find_lru(tree, start, len);
  2315. spin_unlock(&tree->lru_lock);
  2316. if (eb) {
  2317. return eb;
  2318. }
  2319. eb = kmem_cache_zalloc(extent_buffer_cache, mask);
  2320. INIT_LIST_HEAD(&eb->lru);
  2321. eb->start = start;
  2322. eb->len = len;
  2323. atomic_set(&eb->refs, 1);
  2324. return eb;
  2325. }
  2326. static void __free_extent_buffer(struct extent_buffer *eb)
  2327. {
  2328. kmem_cache_free(extent_buffer_cache, eb);
  2329. }
  2330. struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
  2331. u64 start, unsigned long len,
  2332. struct page *page0,
  2333. gfp_t mask)
  2334. {
  2335. unsigned long num_pages = num_extent_pages(start, len);
  2336. unsigned long i;
  2337. unsigned long index = start >> PAGE_CACHE_SHIFT;
  2338. struct extent_buffer *eb;
  2339. struct page *p;
  2340. struct address_space *mapping = tree->mapping;
  2341. int uptodate = 1;
  2342. eb = __alloc_extent_buffer(tree, start, len, mask);
  2343. if (!eb || IS_ERR(eb))
  2344. return NULL;
  2345. if (eb->flags & EXTENT_BUFFER_FILLED)
  2346. goto lru_add;
  2347. if (page0) {
  2348. eb->first_page = page0;
  2349. i = 1;
  2350. index++;
  2351. page_cache_get(page0);
  2352. mark_page_accessed(page0);
  2353. set_page_extent_mapped(page0);
  2354. WARN_ON(!PageUptodate(page0));
  2355. set_page_extent_head(page0, len);
  2356. } else {
  2357. i = 0;
  2358. }
  2359. for (; i < num_pages; i++, index++) {
  2360. p = find_or_create_page(mapping, index, mask | __GFP_HIGHMEM);
  2361. if (!p) {
  2362. WARN_ON(1);
  2363. goto fail;
  2364. }
  2365. set_page_extent_mapped(p);
  2366. mark_page_accessed(p);
  2367. if (i == 0) {
  2368. eb->first_page = p;
  2369. set_page_extent_head(p, len);
  2370. } else {
  2371. set_page_private(p, EXTENT_PAGE_PRIVATE);
  2372. }
  2373. if (!PageUptodate(p))
  2374. uptodate = 0;
  2375. unlock_page(p);
  2376. }
  2377. if (uptodate)
  2378. eb->flags |= EXTENT_UPTODATE;
  2379. eb->flags |= EXTENT_BUFFER_FILLED;
  2380. lru_add:
  2381. spin_lock(&tree->lru_lock);
  2382. add_lru(tree, eb);
  2383. spin_unlock(&tree->lru_lock);
  2384. return eb;
  2385. fail:
  2386. spin_lock(&tree->lru_lock);
  2387. list_del_init(&eb->lru);
  2388. spin_unlock(&tree->lru_lock);
  2389. if (!atomic_dec_and_test(&eb->refs))
  2390. return NULL;
  2391. for (index = 1; index < i; index++) {
  2392. page_cache_release(extent_buffer_page(eb, index));
  2393. }
  2394. if (i > 0)
  2395. page_cache_release(extent_buffer_page(eb, 0));
  2396. __free_extent_buffer(eb);
  2397. return NULL;
  2398. }
  2399. EXPORT_SYMBOL(alloc_extent_buffer);
  2400. struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
  2401. u64 start, unsigned long len,
  2402. gfp_t mask)
  2403. {
  2404. unsigned long num_pages = num_extent_pages(start, len);
  2405. unsigned long i;
  2406. unsigned long index = start >> PAGE_CACHE_SHIFT;
  2407. struct extent_buffer *eb;
  2408. struct page *p;
  2409. struct address_space *mapping = tree->mapping;
  2410. int uptodate = 1;
  2411. eb = __alloc_extent_buffer(tree, start, len, mask);
  2412. if (!eb || IS_ERR(eb))
  2413. return NULL;
  2414. if (eb->flags & EXTENT_BUFFER_FILLED)
  2415. goto lru_add;
  2416. for (i = 0; i < num_pages; i++, index++) {
  2417. p = find_lock_page(mapping, index);
  2418. if (!p) {
  2419. goto fail;
  2420. }
  2421. set_page_extent_mapped(p);
  2422. mark_page_accessed(p);
  2423. if (i == 0) {
  2424. eb->first_page = p;
  2425. set_page_extent_head(p, len);
  2426. } else {
  2427. set_page_private(p, EXTENT_PAGE_PRIVATE);
  2428. }
  2429. if (!PageUptodate(p))
  2430. uptodate = 0;
  2431. unlock_page(p);
  2432. }
  2433. if (uptodate)
  2434. eb->flags |= EXTENT_UPTODATE;
  2435. eb->flags |= EXTENT_BUFFER_FILLED;
  2436. lru_add:
  2437. spin_lock(&tree->lru_lock);
  2438. add_lru(tree, eb);
  2439. spin_unlock(&tree->lru_lock);
  2440. return eb;
  2441. fail:
  2442. spin_lock(&tree->lru_lock);
  2443. list_del_init(&eb->lru);
  2444. spin_unlock(&tree->lru_lock);
  2445. if (!atomic_dec_and_test(&eb->refs))
  2446. return NULL;
  2447. for (index = 1; index < i; index++) {
  2448. page_cache_release(extent_buffer_page(eb, index));
  2449. }
  2450. if (i > 0)
  2451. page_cache_release(extent_buffer_page(eb, 0));
  2452. __free_extent_buffer(eb);
  2453. return NULL;
  2454. }
  2455. EXPORT_SYMBOL(find_extent_buffer);
  2456. void free_extent_buffer(struct extent_buffer *eb)
  2457. {
  2458. unsigned long i;
  2459. unsigned long num_pages;
  2460. if (!eb)
  2461. return;
  2462. if (!atomic_dec_and_test(&eb->refs))
  2463. return;
  2464. WARN_ON(!list_empty(&eb->lru));
  2465. num_pages = num_extent_pages(eb->start, eb->len);
  2466. for (i = 1; i < num_pages; i++) {
  2467. page_cache_release(extent_buffer_page(eb, i));
  2468. }
  2469. page_cache_release(extent_buffer_page(eb, 0));
  2470. __free_extent_buffer(eb);
  2471. }
  2472. EXPORT_SYMBOL(free_extent_buffer);
  2473. int clear_extent_buffer_dirty(struct extent_io_tree *tree,
  2474. struct extent_buffer *eb)
  2475. {
  2476. int set;
  2477. unsigned long i;
  2478. unsigned long num_pages;
  2479. struct page *page;
  2480. u64 start = eb->start;
  2481. u64 end = start + eb->len - 1;
  2482. set = clear_extent_dirty(tree, start, end, GFP_NOFS);
  2483. num_pages = num_extent_pages(eb->start, eb->len);
  2484. for (i = 0; i < num_pages; i++) {
  2485. page = extent_buffer_page(eb, i);
  2486. lock_page(page);
  2487. if (i == 0)
  2488. set_page_extent_head(page, eb->len);
  2489. else
  2490. set_page_private(page, EXTENT_PAGE_PRIVATE);
  2491. /*
  2492. * if we're on the last page or the first page and the
  2493. * block isn't aligned on a page boundary, do extra checks
  2494. * to make sure we don't clean page that is partially dirty
  2495. */
  2496. if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
  2497. ((i == num_pages - 1) &&
  2498. ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
  2499. start = (u64)page->index << PAGE_CACHE_SHIFT;
  2500. end = start + PAGE_CACHE_SIZE - 1;
  2501. if (test_range_bit(tree, start, end,
  2502. EXTENT_DIRTY, 0)) {
  2503. unlock_page(page);
  2504. continue;
  2505. }
  2506. }
  2507. clear_page_dirty_for_io(page);
  2508. read_lock_irq(&page->mapping->tree_lock);
  2509. if (!PageDirty(page)) {
  2510. radix_tree_tag_clear(&page->mapping->page_tree,
  2511. page_index(page),
  2512. PAGECACHE_TAG_DIRTY);
  2513. }
  2514. read_unlock_irq(&page->mapping->tree_lock);
  2515. unlock_page(page);
  2516. }
  2517. return 0;
  2518. }
  2519. EXPORT_SYMBOL(clear_extent_buffer_dirty);
  2520. int wait_on_extent_buffer_writeback(struct extent_io_tree *tree,
  2521. struct extent_buffer *eb)
  2522. {
  2523. return wait_on_extent_writeback(tree, eb->start,
  2524. eb->start + eb->len - 1);
  2525. }
  2526. EXPORT_SYMBOL(wait_on_extent_buffer_writeback);
  2527. int set_extent_buffer_dirty(struct extent_io_tree *tree,
  2528. struct extent_buffer *eb)
  2529. {
  2530. unsigned long i;
  2531. unsigned long num_pages;
  2532. num_pages = num_extent_pages(eb->start, eb->len);
  2533. for (i = 0; i < num_pages; i++) {
  2534. struct page *page = extent_buffer_page(eb, i);
  2535. /* writepage may need to do something special for the
  2536. * first page, we have to make sure page->private is
  2537. * properly set. releasepage may drop page->private
  2538. * on us if the page isn't already dirty.
  2539. */
  2540. if (i == 0) {
  2541. lock_page(page);
  2542. set_page_extent_head(page, eb->len);
  2543. } else if (PagePrivate(page) &&
  2544. page->private != EXTENT_PAGE_PRIVATE) {
  2545. lock_page(page);
  2546. set_page_extent_mapped(page);
  2547. unlock_page(page);
  2548. }
  2549. __set_page_dirty_nobuffers(extent_buffer_page(eb, i));
  2550. if (i == 0)
  2551. unlock_page(page);
  2552. }
  2553. return set_extent_dirty(tree, eb->start,
  2554. eb->start + eb->len - 1, GFP_NOFS);
  2555. }
  2556. EXPORT_SYMBOL(set_extent_buffer_dirty);
  2557. int set_extent_buffer_uptodate(struct extent_io_tree *tree,
  2558. struct extent_buffer *eb)
  2559. {
  2560. unsigned long i;
  2561. struct page *page;
  2562. unsigned long num_pages;
  2563. num_pages = num_extent_pages(eb->start, eb->len);
  2564. set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
  2565. GFP_NOFS);
  2566. for (i = 0; i < num_pages; i++) {
  2567. page = extent_buffer_page(eb, i);
  2568. if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
  2569. ((i == num_pages - 1) &&
  2570. ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
  2571. check_page_uptodate(tree, page);
  2572. continue;
  2573. }
  2574. SetPageUptodate(page);
  2575. }
  2576. return 0;
  2577. }
  2578. EXPORT_SYMBOL(set_extent_buffer_uptodate);
  2579. int extent_buffer_uptodate(struct extent_io_tree *tree,
  2580. struct extent_buffer *eb)
  2581. {
  2582. if (eb->flags & EXTENT_UPTODATE)
  2583. return 1;
  2584. return test_range_bit(tree, eb->start, eb->start + eb->len - 1,
  2585. EXTENT_UPTODATE, 1);
  2586. }
  2587. EXPORT_SYMBOL(extent_buffer_uptodate);
  2588. int read_extent_buffer_pages(struct extent_io_tree *tree,
  2589. struct extent_buffer *eb,
  2590. u64 start,
  2591. int wait)
  2592. {
  2593. unsigned long i;
  2594. unsigned long start_i;
  2595. struct page *page;
  2596. int err;
  2597. int ret = 0;
  2598. unsigned long num_pages;
  2599. if (eb->flags & EXTENT_UPTODATE)
  2600. return 0;
  2601. if (0 && test_range_bit(tree, eb->start, eb->start + eb->len - 1,
  2602. EXTENT_UPTODATE, 1)) {
  2603. return 0;
  2604. }
  2605. if (start) {
  2606. WARN_ON(start < eb->start);
  2607. start_i = (start >> PAGE_CACHE_SHIFT) -
  2608. (eb->start >> PAGE_CACHE_SHIFT);
  2609. } else {
  2610. start_i = 0;
  2611. }
  2612. num_pages = num_extent_pages(eb->start, eb->len);
  2613. for (i = start_i; i < num_pages; i++) {
  2614. page = extent_buffer_page(eb, i);
  2615. if (PageUptodate(page)) {
  2616. continue;
  2617. }
  2618. if (!wait) {
  2619. if (TestSetPageLocked(page)) {
  2620. continue;
  2621. }
  2622. } else {
  2623. lock_page(page);
  2624. }
  2625. if (!PageUptodate(page)) {
  2626. err = page->mapping->a_ops->readpage(NULL, page);
  2627. if (err) {
  2628. ret = err;
  2629. }
  2630. } else {
  2631. unlock_page(page);
  2632. }
  2633. }
  2634. if (ret || !wait) {
  2635. return ret;
  2636. }
  2637. for (i = start_i; i < num_pages; i++) {
  2638. page = extent_buffer_page(eb, i);
  2639. wait_on_page_locked(page);
  2640. if (!PageUptodate(page)) {
  2641. ret = -EIO;
  2642. }
  2643. }
  2644. if (!ret)
  2645. eb->flags |= EXTENT_UPTODATE;
  2646. return ret;
  2647. }
  2648. EXPORT_SYMBOL(read_extent_buffer_pages);
  2649. void read_extent_buffer(struct extent_buffer *eb, void *dstv,
  2650. unsigned long start,
  2651. unsigned long len)
  2652. {
  2653. size_t cur;
  2654. size_t offset;
  2655. struct page *page;
  2656. char *kaddr;
  2657. char *dst = (char *)dstv;
  2658. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2659. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2660. unsigned long num_pages = num_extent_pages(eb->start, eb->len);
  2661. WARN_ON(start > eb->len);
  2662. WARN_ON(start + len > eb->start + eb->len);
  2663. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2664. while(len > 0) {
  2665. page = extent_buffer_page(eb, i);
  2666. if (!PageUptodate(page)) {
  2667. printk("page %lu not up to date i %lu, total %lu, len %lu\n", page->index, i, num_pages, eb->len);
  2668. WARN_ON(1);
  2669. }
  2670. WARN_ON(!PageUptodate(page));
  2671. cur = min(len, (PAGE_CACHE_SIZE - offset));
  2672. kaddr = kmap_atomic(page, KM_USER1);
  2673. memcpy(dst, kaddr + offset, cur);
  2674. kunmap_atomic(kaddr, KM_USER1);
  2675. dst += cur;
  2676. len -= cur;
  2677. offset = 0;
  2678. i++;
  2679. }
  2680. }
  2681. EXPORT_SYMBOL(read_extent_buffer);
  2682. int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
  2683. unsigned long min_len, char **token, char **map,
  2684. unsigned long *map_start,
  2685. unsigned long *map_len, int km)
  2686. {
  2687. size_t offset = start & (PAGE_CACHE_SIZE - 1);
  2688. char *kaddr;
  2689. struct page *p;
  2690. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2691. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2692. unsigned long end_i = (start_offset + start + min_len - 1) >>
  2693. PAGE_CACHE_SHIFT;
  2694. if (i != end_i)
  2695. return -EINVAL;
  2696. if (i == 0) {
  2697. offset = start_offset;
  2698. *map_start = 0;
  2699. } else {
  2700. offset = 0;
  2701. *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
  2702. }
  2703. if (start + min_len > eb->len) {
  2704. printk("bad mapping eb start %Lu len %lu, wanted %lu %lu\n", eb->start, eb->len, start, min_len);
  2705. WARN_ON(1);
  2706. }
  2707. p = extent_buffer_page(eb, i);
  2708. WARN_ON(!PageUptodate(p));
  2709. kaddr = kmap_atomic(p, km);
  2710. *token = kaddr;
  2711. *map = kaddr + offset;
  2712. *map_len = PAGE_CACHE_SIZE - offset;
  2713. return 0;
  2714. }
  2715. EXPORT_SYMBOL(map_private_extent_buffer);
  2716. int map_extent_buffer(struct extent_buffer *eb, unsigned long start,
  2717. unsigned long min_len,
  2718. char **token, char **map,
  2719. unsigned long *map_start,
  2720. unsigned long *map_len, int km)
  2721. {
  2722. int err;
  2723. int save = 0;
  2724. if (eb->map_token) {
  2725. unmap_extent_buffer(eb, eb->map_token, km);
  2726. eb->map_token = NULL;
  2727. save = 1;
  2728. }
  2729. err = map_private_extent_buffer(eb, start, min_len, token, map,
  2730. map_start, map_len, km);
  2731. if (!err && save) {
  2732. eb->map_token = *token;
  2733. eb->kaddr = *map;
  2734. eb->map_start = *map_start;
  2735. eb->map_len = *map_len;
  2736. }
  2737. return err;
  2738. }
  2739. EXPORT_SYMBOL(map_extent_buffer);
  2740. void unmap_extent_buffer(struct extent_buffer *eb, char *token, int km)
  2741. {
  2742. kunmap_atomic(token, km);
  2743. }
  2744. EXPORT_SYMBOL(unmap_extent_buffer);
  2745. int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
  2746. unsigned long start,
  2747. unsigned long len)
  2748. {
  2749. size_t cur;
  2750. size_t offset;
  2751. struct page *page;
  2752. char *kaddr;
  2753. char *ptr = (char *)ptrv;
  2754. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2755. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2756. int ret = 0;
  2757. WARN_ON(start > eb->len);
  2758. WARN_ON(start + len > eb->start + eb->len);
  2759. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2760. while(len > 0) {
  2761. page = extent_buffer_page(eb, i);
  2762. WARN_ON(!PageUptodate(page));
  2763. cur = min(len, (PAGE_CACHE_SIZE - offset));
  2764. kaddr = kmap_atomic(page, KM_USER0);
  2765. ret = memcmp(ptr, kaddr + offset, cur);
  2766. kunmap_atomic(kaddr, KM_USER0);
  2767. if (ret)
  2768. break;
  2769. ptr += cur;
  2770. len -= cur;
  2771. offset = 0;
  2772. i++;
  2773. }
  2774. return ret;
  2775. }
  2776. EXPORT_SYMBOL(memcmp_extent_buffer);
  2777. void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
  2778. unsigned long start, unsigned long len)
  2779. {
  2780. size_t cur;
  2781. size_t offset;
  2782. struct page *page;
  2783. char *kaddr;
  2784. char *src = (char *)srcv;
  2785. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2786. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2787. WARN_ON(start > eb->len);
  2788. WARN_ON(start + len > eb->start + eb->len);
  2789. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2790. while(len > 0) {
  2791. page = extent_buffer_page(eb, i);
  2792. WARN_ON(!PageUptodate(page));
  2793. cur = min(len, PAGE_CACHE_SIZE - offset);
  2794. kaddr = kmap_atomic(page, KM_USER1);
  2795. memcpy(kaddr + offset, src, cur);
  2796. kunmap_atomic(kaddr, KM_USER1);
  2797. src += cur;
  2798. len -= cur;
  2799. offset = 0;
  2800. i++;
  2801. }
  2802. }
  2803. EXPORT_SYMBOL(write_extent_buffer);
  2804. void memset_extent_buffer(struct extent_buffer *eb, char c,
  2805. unsigned long start, unsigned long len)
  2806. {
  2807. size_t cur;
  2808. size_t offset;
  2809. struct page *page;
  2810. char *kaddr;
  2811. size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
  2812. unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
  2813. WARN_ON(start > eb->len);
  2814. WARN_ON(start + len > eb->start + eb->len);
  2815. offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
  2816. while(len > 0) {
  2817. page = extent_buffer_page(eb, i);
  2818. WARN_ON(!PageUptodate(page));
  2819. cur = min(len, PAGE_CACHE_SIZE - offset);
  2820. kaddr = kmap_atomic(page, KM_USER0);
  2821. memset(kaddr + offset, c, cur);
  2822. kunmap_atomic(kaddr, KM_USER0);
  2823. len -= cur;
  2824. offset = 0;
  2825. i++;
  2826. }
  2827. }
  2828. EXPORT_SYMBOL(memset_extent_buffer);
  2829. void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
  2830. unsigned long dst_offset, unsigned long src_offset,
  2831. unsigned long len)
  2832. {
  2833. u64 dst_len = dst->len;
  2834. size_t cur;
  2835. size_t offset;
  2836. struct page *page;
  2837. char *kaddr;
  2838. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  2839. unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
  2840. WARN_ON(src->len != dst_len);
  2841. offset = (start_offset + dst_offset) &
  2842. ((unsigned long)PAGE_CACHE_SIZE - 1);
  2843. while(len > 0) {
  2844. page = extent_buffer_page(dst, i);
  2845. WARN_ON(!PageUptodate(page));
  2846. cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
  2847. kaddr = kmap_atomic(page, KM_USER0);
  2848. read_extent_buffer(src, kaddr + offset, src_offset, cur);
  2849. kunmap_atomic(kaddr, KM_USER0);
  2850. src_offset += cur;
  2851. len -= cur;
  2852. offset = 0;
  2853. i++;
  2854. }
  2855. }
  2856. EXPORT_SYMBOL(copy_extent_buffer);
  2857. static void move_pages(struct page *dst_page, struct page *src_page,
  2858. unsigned long dst_off, unsigned long src_off,
  2859. unsigned long len)
  2860. {
  2861. char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
  2862. if (dst_page == src_page) {
  2863. memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
  2864. } else {
  2865. char *src_kaddr = kmap_atomic(src_page, KM_USER1);
  2866. char *p = dst_kaddr + dst_off + len;
  2867. char *s = src_kaddr + src_off + len;
  2868. while (len--)
  2869. *--p = *--s;
  2870. kunmap_atomic(src_kaddr, KM_USER1);
  2871. }
  2872. kunmap_atomic(dst_kaddr, KM_USER0);
  2873. }
  2874. static void copy_pages(struct page *dst_page, struct page *src_page,
  2875. unsigned long dst_off, unsigned long src_off,
  2876. unsigned long len)
  2877. {
  2878. char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
  2879. char *src_kaddr;
  2880. if (dst_page != src_page)
  2881. src_kaddr = kmap_atomic(src_page, KM_USER1);
  2882. else
  2883. src_kaddr = dst_kaddr;
  2884. memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
  2885. kunmap_atomic(dst_kaddr, KM_USER0);
  2886. if (dst_page != src_page)
  2887. kunmap_atomic(src_kaddr, KM_USER1);
  2888. }
  2889. void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
  2890. unsigned long src_offset, unsigned long len)
  2891. {
  2892. size_t cur;
  2893. size_t dst_off_in_page;
  2894. size_t src_off_in_page;
  2895. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  2896. unsigned long dst_i;
  2897. unsigned long src_i;
  2898. if (src_offset + len > dst->len) {
  2899. printk("memmove bogus src_offset %lu move len %lu len %lu\n",
  2900. src_offset, len, dst->len);
  2901. BUG_ON(1);
  2902. }
  2903. if (dst_offset + len > dst->len) {
  2904. printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
  2905. dst_offset, len, dst->len);
  2906. BUG_ON(1);
  2907. }
  2908. while(len > 0) {
  2909. dst_off_in_page = (start_offset + dst_offset) &
  2910. ((unsigned long)PAGE_CACHE_SIZE - 1);
  2911. src_off_in_page = (start_offset + src_offset) &
  2912. ((unsigned long)PAGE_CACHE_SIZE - 1);
  2913. dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
  2914. src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
  2915. cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
  2916. src_off_in_page));
  2917. cur = min_t(unsigned long, cur,
  2918. (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
  2919. copy_pages(extent_buffer_page(dst, dst_i),
  2920. extent_buffer_page(dst, src_i),
  2921. dst_off_in_page, src_off_in_page, cur);
  2922. src_offset += cur;
  2923. dst_offset += cur;
  2924. len -= cur;
  2925. }
  2926. }
  2927. EXPORT_SYMBOL(memcpy_extent_buffer);
  2928. void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
  2929. unsigned long src_offset, unsigned long len)
  2930. {
  2931. size_t cur;
  2932. size_t dst_off_in_page;
  2933. size_t src_off_in_page;
  2934. unsigned long dst_end = dst_offset + len - 1;
  2935. unsigned long src_end = src_offset + len - 1;
  2936. size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
  2937. unsigned long dst_i;
  2938. unsigned long src_i;
  2939. if (src_offset + len > dst->len) {
  2940. printk("memmove bogus src_offset %lu move len %lu len %lu\n",
  2941. src_offset, len, dst->len);
  2942. BUG_ON(1);
  2943. }
  2944. if (dst_offset + len > dst->len) {
  2945. printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
  2946. dst_offset, len, dst->len);
  2947. BUG_ON(1);
  2948. }
  2949. if (dst_offset < src_offset) {
  2950. memcpy_extent_buffer(dst, dst_offset, src_offset, len);
  2951. return;
  2952. }
  2953. while(len > 0) {
  2954. dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
  2955. src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
  2956. dst_off_in_page = (start_offset + dst_end) &
  2957. ((unsigned long)PAGE_CACHE_SIZE - 1);
  2958. src_off_in_page = (start_offset + src_end) &
  2959. ((unsigned long)PAGE_CACHE_SIZE - 1);
  2960. cur = min_t(unsigned long, len, src_off_in_page + 1);
  2961. cur = min(cur, dst_off_in_page + 1);
  2962. move_pages(extent_buffer_page(dst, dst_i),
  2963. extent_buffer_page(dst, src_i),
  2964. dst_off_in_page - cur + 1,
  2965. src_off_in_page - cur + 1, cur);
  2966. dst_end -= cur;
  2967. src_end -= cur;
  2968. len -= cur;
  2969. }
  2970. }
  2971. EXPORT_SYMBOL(memmove_extent_buffer);