file.c 48 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886
  1. /*
  2. * Copyright (C) 2007 Oracle. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public
  6. * License v2 as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public
  14. * License along with this program; if not, write to the
  15. * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16. * Boston, MA 021110-1307, USA.
  17. */
  18. #include <linux/fs.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/highmem.h>
  21. #include <linux/time.h>
  22. #include <linux/init.h>
  23. #include <linux/string.h>
  24. #include <linux/backing-dev.h>
  25. #include <linux/mpage.h>
  26. #include <linux/falloc.h>
  27. #include <linux/swap.h>
  28. #include <linux/writeback.h>
  29. #include <linux/statfs.h>
  30. #include <linux/compat.h>
  31. #include <linux/slab.h>
  32. #include "ctree.h"
  33. #include "disk-io.h"
  34. #include "transaction.h"
  35. #include "btrfs_inode.h"
  36. #include "ioctl.h"
  37. #include "print-tree.h"
  38. #include "tree-log.h"
  39. #include "locking.h"
  40. #include "compat.h"
  41. /*
  42. * when auto defrag is enabled we
  43. * queue up these defrag structs to remember which
  44. * inodes need defragging passes
  45. */
  46. struct inode_defrag {
  47. struct rb_node rb_node;
  48. /* objectid */
  49. u64 ino;
  50. /*
  51. * transid where the defrag was added, we search for
  52. * extents newer than this
  53. */
  54. u64 transid;
  55. /* root objectid */
  56. u64 root;
  57. /* last offset we were able to defrag */
  58. u64 last_offset;
  59. /* if we've wrapped around back to zero once already */
  60. int cycled;
  61. };
  62. /* pop a record for an inode into the defrag tree. The lock
  63. * must be held already
  64. *
  65. * If you're inserting a record for an older transid than an
  66. * existing record, the transid already in the tree is lowered
  67. *
  68. * If an existing record is found the defrag item you
  69. * pass in is freed
  70. */
  71. static void __btrfs_add_inode_defrag(struct inode *inode,
  72. struct inode_defrag *defrag)
  73. {
  74. struct btrfs_root *root = BTRFS_I(inode)->root;
  75. struct inode_defrag *entry;
  76. struct rb_node **p;
  77. struct rb_node *parent = NULL;
  78. p = &root->fs_info->defrag_inodes.rb_node;
  79. while (*p) {
  80. parent = *p;
  81. entry = rb_entry(parent, struct inode_defrag, rb_node);
  82. if (defrag->ino < entry->ino)
  83. p = &parent->rb_left;
  84. else if (defrag->ino > entry->ino)
  85. p = &parent->rb_right;
  86. else {
  87. /* if we're reinserting an entry for
  88. * an old defrag run, make sure to
  89. * lower the transid of our existing record
  90. */
  91. if (defrag->transid < entry->transid)
  92. entry->transid = defrag->transid;
  93. if (defrag->last_offset > entry->last_offset)
  94. entry->last_offset = defrag->last_offset;
  95. goto exists;
  96. }
  97. }
  98. BTRFS_I(inode)->in_defrag = 1;
  99. rb_link_node(&defrag->rb_node, parent, p);
  100. rb_insert_color(&defrag->rb_node, &root->fs_info->defrag_inodes);
  101. return;
  102. exists:
  103. kfree(defrag);
  104. return;
  105. }
  106. /*
  107. * insert a defrag record for this inode if auto defrag is
  108. * enabled
  109. */
  110. int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
  111. struct inode *inode)
  112. {
  113. struct btrfs_root *root = BTRFS_I(inode)->root;
  114. struct inode_defrag *defrag;
  115. u64 transid;
  116. if (!btrfs_test_opt(root, AUTO_DEFRAG))
  117. return 0;
  118. if (btrfs_fs_closing(root->fs_info))
  119. return 0;
  120. if (BTRFS_I(inode)->in_defrag)
  121. return 0;
  122. if (trans)
  123. transid = trans->transid;
  124. else
  125. transid = BTRFS_I(inode)->root->last_trans;
  126. defrag = kzalloc(sizeof(*defrag), GFP_NOFS);
  127. if (!defrag)
  128. return -ENOMEM;
  129. defrag->ino = btrfs_ino(inode);
  130. defrag->transid = transid;
  131. defrag->root = root->root_key.objectid;
  132. spin_lock(&root->fs_info->defrag_inodes_lock);
  133. if (!BTRFS_I(inode)->in_defrag)
  134. __btrfs_add_inode_defrag(inode, defrag);
  135. else
  136. kfree(defrag);
  137. spin_unlock(&root->fs_info->defrag_inodes_lock);
  138. return 0;
  139. }
  140. /*
  141. * must be called with the defrag_inodes lock held
  142. */
  143. struct inode_defrag *btrfs_find_defrag_inode(struct btrfs_fs_info *info, u64 ino,
  144. struct rb_node **next)
  145. {
  146. struct inode_defrag *entry = NULL;
  147. struct rb_node *p;
  148. struct rb_node *parent = NULL;
  149. p = info->defrag_inodes.rb_node;
  150. while (p) {
  151. parent = p;
  152. entry = rb_entry(parent, struct inode_defrag, rb_node);
  153. if (ino < entry->ino)
  154. p = parent->rb_left;
  155. else if (ino > entry->ino)
  156. p = parent->rb_right;
  157. else
  158. return entry;
  159. }
  160. if (next) {
  161. while (parent && ino > entry->ino) {
  162. parent = rb_next(parent);
  163. entry = rb_entry(parent, struct inode_defrag, rb_node);
  164. }
  165. *next = parent;
  166. }
  167. return NULL;
  168. }
  169. /*
  170. * run through the list of inodes in the FS that need
  171. * defragging
  172. */
  173. int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
  174. {
  175. struct inode_defrag *defrag;
  176. struct btrfs_root *inode_root;
  177. struct inode *inode;
  178. struct rb_node *n;
  179. struct btrfs_key key;
  180. struct btrfs_ioctl_defrag_range_args range;
  181. u64 first_ino = 0;
  182. int num_defrag;
  183. int defrag_batch = 1024;
  184. memset(&range, 0, sizeof(range));
  185. range.len = (u64)-1;
  186. atomic_inc(&fs_info->defrag_running);
  187. spin_lock(&fs_info->defrag_inodes_lock);
  188. while(1) {
  189. n = NULL;
  190. /* find an inode to defrag */
  191. defrag = btrfs_find_defrag_inode(fs_info, first_ino, &n);
  192. if (!defrag) {
  193. if (n)
  194. defrag = rb_entry(n, struct inode_defrag, rb_node);
  195. else if (first_ino) {
  196. first_ino = 0;
  197. continue;
  198. } else {
  199. break;
  200. }
  201. }
  202. /* remove it from the rbtree */
  203. first_ino = defrag->ino + 1;
  204. rb_erase(&defrag->rb_node, &fs_info->defrag_inodes);
  205. if (btrfs_fs_closing(fs_info))
  206. goto next_free;
  207. spin_unlock(&fs_info->defrag_inodes_lock);
  208. /* get the inode */
  209. key.objectid = defrag->root;
  210. btrfs_set_key_type(&key, BTRFS_ROOT_ITEM_KEY);
  211. key.offset = (u64)-1;
  212. inode_root = btrfs_read_fs_root_no_name(fs_info, &key);
  213. if (IS_ERR(inode_root))
  214. goto next;
  215. key.objectid = defrag->ino;
  216. btrfs_set_key_type(&key, BTRFS_INODE_ITEM_KEY);
  217. key.offset = 0;
  218. inode = btrfs_iget(fs_info->sb, &key, inode_root, NULL);
  219. if (IS_ERR(inode))
  220. goto next;
  221. /* do a chunk of defrag */
  222. BTRFS_I(inode)->in_defrag = 0;
  223. range.start = defrag->last_offset;
  224. num_defrag = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
  225. defrag_batch);
  226. /*
  227. * if we filled the whole defrag batch, there
  228. * must be more work to do. Queue this defrag
  229. * again
  230. */
  231. if (num_defrag == defrag_batch) {
  232. defrag->last_offset = range.start;
  233. __btrfs_add_inode_defrag(inode, defrag);
  234. /*
  235. * we don't want to kfree defrag, we added it back to
  236. * the rbtree
  237. */
  238. defrag = NULL;
  239. } else if (defrag->last_offset && !defrag->cycled) {
  240. /*
  241. * we didn't fill our defrag batch, but
  242. * we didn't start at zero. Make sure we loop
  243. * around to the start of the file.
  244. */
  245. defrag->last_offset = 0;
  246. defrag->cycled = 1;
  247. __btrfs_add_inode_defrag(inode, defrag);
  248. defrag = NULL;
  249. }
  250. iput(inode);
  251. next:
  252. spin_lock(&fs_info->defrag_inodes_lock);
  253. next_free:
  254. kfree(defrag);
  255. }
  256. spin_unlock(&fs_info->defrag_inodes_lock);
  257. atomic_dec(&fs_info->defrag_running);
  258. /*
  259. * during unmount, we use the transaction_wait queue to
  260. * wait for the defragger to stop
  261. */
  262. wake_up(&fs_info->transaction_wait);
  263. return 0;
  264. }
  265. /* simple helper to fault in pages and copy. This should go away
  266. * and be replaced with calls into generic code.
  267. */
  268. static noinline int btrfs_copy_from_user(loff_t pos, int num_pages,
  269. size_t write_bytes,
  270. struct page **prepared_pages,
  271. struct iov_iter *i)
  272. {
  273. size_t copied = 0;
  274. size_t total_copied = 0;
  275. int pg = 0;
  276. int offset = pos & (PAGE_CACHE_SIZE - 1);
  277. while (write_bytes > 0) {
  278. size_t count = min_t(size_t,
  279. PAGE_CACHE_SIZE - offset, write_bytes);
  280. struct page *page = prepared_pages[pg];
  281. /*
  282. * Copy data from userspace to the current page
  283. *
  284. * Disable pagefault to avoid recursive lock since
  285. * the pages are already locked
  286. */
  287. pagefault_disable();
  288. copied = iov_iter_copy_from_user_atomic(page, i, offset, count);
  289. pagefault_enable();
  290. /* Flush processor's dcache for this page */
  291. flush_dcache_page(page);
  292. /*
  293. * if we get a partial write, we can end up with
  294. * partially up to date pages. These add
  295. * a lot of complexity, so make sure they don't
  296. * happen by forcing this copy to be retried.
  297. *
  298. * The rest of the btrfs_file_write code will fall
  299. * back to page at a time copies after we return 0.
  300. */
  301. if (!PageUptodate(page) && copied < count)
  302. copied = 0;
  303. iov_iter_advance(i, copied);
  304. write_bytes -= copied;
  305. total_copied += copied;
  306. /* Return to btrfs_file_aio_write to fault page */
  307. if (unlikely(copied == 0))
  308. break;
  309. if (unlikely(copied < PAGE_CACHE_SIZE - offset)) {
  310. offset += copied;
  311. } else {
  312. pg++;
  313. offset = 0;
  314. }
  315. }
  316. return total_copied;
  317. }
  318. /*
  319. * unlocks pages after btrfs_file_write is done with them
  320. */
  321. void btrfs_drop_pages(struct page **pages, size_t num_pages)
  322. {
  323. size_t i;
  324. for (i = 0; i < num_pages; i++) {
  325. /* page checked is some magic around finding pages that
  326. * have been modified without going through btrfs_set_page_dirty
  327. * clear it here
  328. */
  329. ClearPageChecked(pages[i]);
  330. unlock_page(pages[i]);
  331. mark_page_accessed(pages[i]);
  332. page_cache_release(pages[i]);
  333. }
  334. }
  335. /*
  336. * after copy_from_user, pages need to be dirtied and we need to make
  337. * sure holes are created between the current EOF and the start of
  338. * any next extents (if required).
  339. *
  340. * this also makes the decision about creating an inline extent vs
  341. * doing real data extents, marking pages dirty and delalloc as required.
  342. */
  343. int btrfs_dirty_pages(struct btrfs_root *root, struct inode *inode,
  344. struct page **pages, size_t num_pages,
  345. loff_t pos, size_t write_bytes,
  346. struct extent_state **cached)
  347. {
  348. int err = 0;
  349. int i;
  350. u64 num_bytes;
  351. u64 start_pos;
  352. u64 end_of_last_block;
  353. u64 end_pos = pos + write_bytes;
  354. loff_t isize = i_size_read(inode);
  355. start_pos = pos & ~((u64)root->sectorsize - 1);
  356. num_bytes = (write_bytes + pos - start_pos +
  357. root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
  358. end_of_last_block = start_pos + num_bytes - 1;
  359. err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
  360. cached);
  361. if (err)
  362. return err;
  363. for (i = 0; i < num_pages; i++) {
  364. struct page *p = pages[i];
  365. SetPageUptodate(p);
  366. ClearPageChecked(p);
  367. set_page_dirty(p);
  368. }
  369. /*
  370. * we've only changed i_size in ram, and we haven't updated
  371. * the disk i_size. There is no need to log the inode
  372. * at this time.
  373. */
  374. if (end_pos > isize)
  375. i_size_write(inode, end_pos);
  376. return 0;
  377. }
  378. /*
  379. * this drops all the extents in the cache that intersect the range
  380. * [start, end]. Existing extents are split as required.
  381. */
  382. int btrfs_drop_extent_cache(struct inode *inode, u64 start, u64 end,
  383. int skip_pinned)
  384. {
  385. struct extent_map *em;
  386. struct extent_map *split = NULL;
  387. struct extent_map *split2 = NULL;
  388. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  389. u64 len = end - start + 1;
  390. int ret;
  391. int testend = 1;
  392. unsigned long flags;
  393. int compressed = 0;
  394. WARN_ON(end < start);
  395. if (end == (u64)-1) {
  396. len = (u64)-1;
  397. testend = 0;
  398. }
  399. while (1) {
  400. if (!split)
  401. split = alloc_extent_map();
  402. if (!split2)
  403. split2 = alloc_extent_map();
  404. BUG_ON(!split || !split2);
  405. write_lock(&em_tree->lock);
  406. em = lookup_extent_mapping(em_tree, start, len);
  407. if (!em) {
  408. write_unlock(&em_tree->lock);
  409. break;
  410. }
  411. flags = em->flags;
  412. if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
  413. if (testend && em->start + em->len >= start + len) {
  414. free_extent_map(em);
  415. write_unlock(&em_tree->lock);
  416. break;
  417. }
  418. start = em->start + em->len;
  419. if (testend)
  420. len = start + len - (em->start + em->len);
  421. free_extent_map(em);
  422. write_unlock(&em_tree->lock);
  423. continue;
  424. }
  425. compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  426. clear_bit(EXTENT_FLAG_PINNED, &em->flags);
  427. remove_extent_mapping(em_tree, em);
  428. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  429. em->start < start) {
  430. split->start = em->start;
  431. split->len = start - em->start;
  432. split->orig_start = em->orig_start;
  433. split->block_start = em->block_start;
  434. if (compressed)
  435. split->block_len = em->block_len;
  436. else
  437. split->block_len = split->len;
  438. split->bdev = em->bdev;
  439. split->flags = flags;
  440. split->compress_type = em->compress_type;
  441. ret = add_extent_mapping(em_tree, split);
  442. BUG_ON(ret);
  443. free_extent_map(split);
  444. split = split2;
  445. split2 = NULL;
  446. }
  447. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  448. testend && em->start + em->len > start + len) {
  449. u64 diff = start + len - em->start;
  450. split->start = start + len;
  451. split->len = em->start + em->len - (start + len);
  452. split->bdev = em->bdev;
  453. split->flags = flags;
  454. split->compress_type = em->compress_type;
  455. if (compressed) {
  456. split->block_len = em->block_len;
  457. split->block_start = em->block_start;
  458. split->orig_start = em->orig_start;
  459. } else {
  460. split->block_len = split->len;
  461. split->block_start = em->block_start + diff;
  462. split->orig_start = split->start;
  463. }
  464. ret = add_extent_mapping(em_tree, split);
  465. BUG_ON(ret);
  466. free_extent_map(split);
  467. split = NULL;
  468. }
  469. write_unlock(&em_tree->lock);
  470. /* once for us */
  471. free_extent_map(em);
  472. /* once for the tree*/
  473. free_extent_map(em);
  474. }
  475. if (split)
  476. free_extent_map(split);
  477. if (split2)
  478. free_extent_map(split2);
  479. return 0;
  480. }
  481. /*
  482. * this is very complex, but the basic idea is to drop all extents
  483. * in the range start - end. hint_block is filled in with a block number
  484. * that would be a good hint to the block allocator for this file.
  485. *
  486. * If an extent intersects the range but is not entirely inside the range
  487. * it is either truncated or split. Anything entirely inside the range
  488. * is deleted from the tree.
  489. */
  490. int btrfs_drop_extents(struct btrfs_trans_handle *trans, struct inode *inode,
  491. u64 start, u64 end, u64 *hint_byte, int drop_cache)
  492. {
  493. struct btrfs_root *root = BTRFS_I(inode)->root;
  494. struct extent_buffer *leaf;
  495. struct btrfs_file_extent_item *fi;
  496. struct btrfs_path *path;
  497. struct btrfs_key key;
  498. struct btrfs_key new_key;
  499. u64 ino = btrfs_ino(inode);
  500. u64 search_start = start;
  501. u64 disk_bytenr = 0;
  502. u64 num_bytes = 0;
  503. u64 extent_offset = 0;
  504. u64 extent_end = 0;
  505. int del_nr = 0;
  506. int del_slot = 0;
  507. int extent_type;
  508. int recow;
  509. int ret;
  510. if (drop_cache)
  511. btrfs_drop_extent_cache(inode, start, end - 1, 0);
  512. path = btrfs_alloc_path();
  513. if (!path)
  514. return -ENOMEM;
  515. while (1) {
  516. recow = 0;
  517. ret = btrfs_lookup_file_extent(trans, root, path, ino,
  518. search_start, -1);
  519. if (ret < 0)
  520. break;
  521. if (ret > 0 && path->slots[0] > 0 && search_start == start) {
  522. leaf = path->nodes[0];
  523. btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
  524. if (key.objectid == ino &&
  525. key.type == BTRFS_EXTENT_DATA_KEY)
  526. path->slots[0]--;
  527. }
  528. ret = 0;
  529. next_slot:
  530. leaf = path->nodes[0];
  531. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  532. BUG_ON(del_nr > 0);
  533. ret = btrfs_next_leaf(root, path);
  534. if (ret < 0)
  535. break;
  536. if (ret > 0) {
  537. ret = 0;
  538. break;
  539. }
  540. leaf = path->nodes[0];
  541. recow = 1;
  542. }
  543. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  544. if (key.objectid > ino ||
  545. key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= end)
  546. break;
  547. fi = btrfs_item_ptr(leaf, path->slots[0],
  548. struct btrfs_file_extent_item);
  549. extent_type = btrfs_file_extent_type(leaf, fi);
  550. if (extent_type == BTRFS_FILE_EXTENT_REG ||
  551. extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  552. disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  553. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  554. extent_offset = btrfs_file_extent_offset(leaf, fi);
  555. extent_end = key.offset +
  556. btrfs_file_extent_num_bytes(leaf, fi);
  557. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  558. extent_end = key.offset +
  559. btrfs_file_extent_inline_len(leaf, fi);
  560. } else {
  561. WARN_ON(1);
  562. extent_end = search_start;
  563. }
  564. if (extent_end <= search_start) {
  565. path->slots[0]++;
  566. goto next_slot;
  567. }
  568. search_start = max(key.offset, start);
  569. if (recow) {
  570. btrfs_release_path(path);
  571. continue;
  572. }
  573. /*
  574. * | - range to drop - |
  575. * | -------- extent -------- |
  576. */
  577. if (start > key.offset && end < extent_end) {
  578. BUG_ON(del_nr > 0);
  579. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  580. memcpy(&new_key, &key, sizeof(new_key));
  581. new_key.offset = start;
  582. ret = btrfs_duplicate_item(trans, root, path,
  583. &new_key);
  584. if (ret == -EAGAIN) {
  585. btrfs_release_path(path);
  586. continue;
  587. }
  588. if (ret < 0)
  589. break;
  590. leaf = path->nodes[0];
  591. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  592. struct btrfs_file_extent_item);
  593. btrfs_set_file_extent_num_bytes(leaf, fi,
  594. start - key.offset);
  595. fi = btrfs_item_ptr(leaf, path->slots[0],
  596. struct btrfs_file_extent_item);
  597. extent_offset += start - key.offset;
  598. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  599. btrfs_set_file_extent_num_bytes(leaf, fi,
  600. extent_end - start);
  601. btrfs_mark_buffer_dirty(leaf);
  602. if (disk_bytenr > 0) {
  603. ret = btrfs_inc_extent_ref(trans, root,
  604. disk_bytenr, num_bytes, 0,
  605. root->root_key.objectid,
  606. new_key.objectid,
  607. start - extent_offset);
  608. BUG_ON(ret);
  609. *hint_byte = disk_bytenr;
  610. }
  611. key.offset = start;
  612. }
  613. /*
  614. * | ---- range to drop ----- |
  615. * | -------- extent -------- |
  616. */
  617. if (start <= key.offset && end < extent_end) {
  618. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  619. memcpy(&new_key, &key, sizeof(new_key));
  620. new_key.offset = end;
  621. btrfs_set_item_key_safe(trans, root, path, &new_key);
  622. extent_offset += end - key.offset;
  623. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  624. btrfs_set_file_extent_num_bytes(leaf, fi,
  625. extent_end - end);
  626. btrfs_mark_buffer_dirty(leaf);
  627. if (disk_bytenr > 0) {
  628. inode_sub_bytes(inode, end - key.offset);
  629. *hint_byte = disk_bytenr;
  630. }
  631. break;
  632. }
  633. search_start = extent_end;
  634. /*
  635. * | ---- range to drop ----- |
  636. * | -------- extent -------- |
  637. */
  638. if (start > key.offset && end >= extent_end) {
  639. BUG_ON(del_nr > 0);
  640. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  641. btrfs_set_file_extent_num_bytes(leaf, fi,
  642. start - key.offset);
  643. btrfs_mark_buffer_dirty(leaf);
  644. if (disk_bytenr > 0) {
  645. inode_sub_bytes(inode, extent_end - start);
  646. *hint_byte = disk_bytenr;
  647. }
  648. if (end == extent_end)
  649. break;
  650. path->slots[0]++;
  651. goto next_slot;
  652. }
  653. /*
  654. * | ---- range to drop ----- |
  655. * | ------ extent ------ |
  656. */
  657. if (start <= key.offset && end >= extent_end) {
  658. if (del_nr == 0) {
  659. del_slot = path->slots[0];
  660. del_nr = 1;
  661. } else {
  662. BUG_ON(del_slot + del_nr != path->slots[0]);
  663. del_nr++;
  664. }
  665. if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  666. inode_sub_bytes(inode,
  667. extent_end - key.offset);
  668. extent_end = ALIGN(extent_end,
  669. root->sectorsize);
  670. } else if (disk_bytenr > 0) {
  671. ret = btrfs_free_extent(trans, root,
  672. disk_bytenr, num_bytes, 0,
  673. root->root_key.objectid,
  674. key.objectid, key.offset -
  675. extent_offset);
  676. BUG_ON(ret);
  677. inode_sub_bytes(inode,
  678. extent_end - key.offset);
  679. *hint_byte = disk_bytenr;
  680. }
  681. if (end == extent_end)
  682. break;
  683. if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
  684. path->slots[0]++;
  685. goto next_slot;
  686. }
  687. ret = btrfs_del_items(trans, root, path, del_slot,
  688. del_nr);
  689. BUG_ON(ret);
  690. del_nr = 0;
  691. del_slot = 0;
  692. btrfs_release_path(path);
  693. continue;
  694. }
  695. BUG_ON(1);
  696. }
  697. if (del_nr > 0) {
  698. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  699. BUG_ON(ret);
  700. }
  701. btrfs_free_path(path);
  702. return ret;
  703. }
  704. static int extent_mergeable(struct extent_buffer *leaf, int slot,
  705. u64 objectid, u64 bytenr, u64 orig_offset,
  706. u64 *start, u64 *end)
  707. {
  708. struct btrfs_file_extent_item *fi;
  709. struct btrfs_key key;
  710. u64 extent_end;
  711. if (slot < 0 || slot >= btrfs_header_nritems(leaf))
  712. return 0;
  713. btrfs_item_key_to_cpu(leaf, &key, slot);
  714. if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
  715. return 0;
  716. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  717. if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
  718. btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
  719. btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
  720. btrfs_file_extent_compression(leaf, fi) ||
  721. btrfs_file_extent_encryption(leaf, fi) ||
  722. btrfs_file_extent_other_encoding(leaf, fi))
  723. return 0;
  724. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  725. if ((*start && *start != key.offset) || (*end && *end != extent_end))
  726. return 0;
  727. *start = key.offset;
  728. *end = extent_end;
  729. return 1;
  730. }
  731. /*
  732. * Mark extent in the range start - end as written.
  733. *
  734. * This changes extent type from 'pre-allocated' to 'regular'. If only
  735. * part of extent is marked as written, the extent will be split into
  736. * two or three.
  737. */
  738. int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
  739. struct inode *inode, u64 start, u64 end)
  740. {
  741. struct btrfs_root *root = BTRFS_I(inode)->root;
  742. struct extent_buffer *leaf;
  743. struct btrfs_path *path;
  744. struct btrfs_file_extent_item *fi;
  745. struct btrfs_key key;
  746. struct btrfs_key new_key;
  747. u64 bytenr;
  748. u64 num_bytes;
  749. u64 extent_end;
  750. u64 orig_offset;
  751. u64 other_start;
  752. u64 other_end;
  753. u64 split;
  754. int del_nr = 0;
  755. int del_slot = 0;
  756. int recow;
  757. int ret;
  758. u64 ino = btrfs_ino(inode);
  759. btrfs_drop_extent_cache(inode, start, end - 1, 0);
  760. path = btrfs_alloc_path();
  761. if (!path)
  762. return -ENOMEM;
  763. again:
  764. recow = 0;
  765. split = start;
  766. key.objectid = ino;
  767. key.type = BTRFS_EXTENT_DATA_KEY;
  768. key.offset = split;
  769. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  770. if (ret < 0)
  771. goto out;
  772. if (ret > 0 && path->slots[0] > 0)
  773. path->slots[0]--;
  774. leaf = path->nodes[0];
  775. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  776. BUG_ON(key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY);
  777. fi = btrfs_item_ptr(leaf, path->slots[0],
  778. struct btrfs_file_extent_item);
  779. BUG_ON(btrfs_file_extent_type(leaf, fi) !=
  780. BTRFS_FILE_EXTENT_PREALLOC);
  781. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  782. BUG_ON(key.offset > start || extent_end < end);
  783. bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  784. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  785. orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
  786. memcpy(&new_key, &key, sizeof(new_key));
  787. if (start == key.offset && end < extent_end) {
  788. other_start = 0;
  789. other_end = start;
  790. if (extent_mergeable(leaf, path->slots[0] - 1,
  791. ino, bytenr, orig_offset,
  792. &other_start, &other_end)) {
  793. new_key.offset = end;
  794. btrfs_set_item_key_safe(trans, root, path, &new_key);
  795. fi = btrfs_item_ptr(leaf, path->slots[0],
  796. struct btrfs_file_extent_item);
  797. btrfs_set_file_extent_num_bytes(leaf, fi,
  798. extent_end - end);
  799. btrfs_set_file_extent_offset(leaf, fi,
  800. end - orig_offset);
  801. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  802. struct btrfs_file_extent_item);
  803. btrfs_set_file_extent_num_bytes(leaf, fi,
  804. end - other_start);
  805. btrfs_mark_buffer_dirty(leaf);
  806. goto out;
  807. }
  808. }
  809. if (start > key.offset && end == extent_end) {
  810. other_start = end;
  811. other_end = 0;
  812. if (extent_mergeable(leaf, path->slots[0] + 1,
  813. ino, bytenr, orig_offset,
  814. &other_start, &other_end)) {
  815. fi = btrfs_item_ptr(leaf, path->slots[0],
  816. struct btrfs_file_extent_item);
  817. btrfs_set_file_extent_num_bytes(leaf, fi,
  818. start - key.offset);
  819. path->slots[0]++;
  820. new_key.offset = start;
  821. btrfs_set_item_key_safe(trans, root, path, &new_key);
  822. fi = btrfs_item_ptr(leaf, path->slots[0],
  823. struct btrfs_file_extent_item);
  824. btrfs_set_file_extent_num_bytes(leaf, fi,
  825. other_end - start);
  826. btrfs_set_file_extent_offset(leaf, fi,
  827. start - orig_offset);
  828. btrfs_mark_buffer_dirty(leaf);
  829. goto out;
  830. }
  831. }
  832. while (start > key.offset || end < extent_end) {
  833. if (key.offset == start)
  834. split = end;
  835. new_key.offset = split;
  836. ret = btrfs_duplicate_item(trans, root, path, &new_key);
  837. if (ret == -EAGAIN) {
  838. btrfs_release_path(path);
  839. goto again;
  840. }
  841. BUG_ON(ret < 0);
  842. leaf = path->nodes[0];
  843. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  844. struct btrfs_file_extent_item);
  845. btrfs_set_file_extent_num_bytes(leaf, fi,
  846. split - key.offset);
  847. fi = btrfs_item_ptr(leaf, path->slots[0],
  848. struct btrfs_file_extent_item);
  849. btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
  850. btrfs_set_file_extent_num_bytes(leaf, fi,
  851. extent_end - split);
  852. btrfs_mark_buffer_dirty(leaf);
  853. ret = btrfs_inc_extent_ref(trans, root, bytenr, num_bytes, 0,
  854. root->root_key.objectid,
  855. ino, orig_offset);
  856. BUG_ON(ret);
  857. if (split == start) {
  858. key.offset = start;
  859. } else {
  860. BUG_ON(start != key.offset);
  861. path->slots[0]--;
  862. extent_end = end;
  863. }
  864. recow = 1;
  865. }
  866. other_start = end;
  867. other_end = 0;
  868. if (extent_mergeable(leaf, path->slots[0] + 1,
  869. ino, bytenr, orig_offset,
  870. &other_start, &other_end)) {
  871. if (recow) {
  872. btrfs_release_path(path);
  873. goto again;
  874. }
  875. extent_end = other_end;
  876. del_slot = path->slots[0] + 1;
  877. del_nr++;
  878. ret = btrfs_free_extent(trans, root, bytenr, num_bytes,
  879. 0, root->root_key.objectid,
  880. ino, orig_offset);
  881. BUG_ON(ret);
  882. }
  883. other_start = 0;
  884. other_end = start;
  885. if (extent_mergeable(leaf, path->slots[0] - 1,
  886. ino, bytenr, orig_offset,
  887. &other_start, &other_end)) {
  888. if (recow) {
  889. btrfs_release_path(path);
  890. goto again;
  891. }
  892. key.offset = other_start;
  893. del_slot = path->slots[0];
  894. del_nr++;
  895. ret = btrfs_free_extent(trans, root, bytenr, num_bytes,
  896. 0, root->root_key.objectid,
  897. ino, orig_offset);
  898. BUG_ON(ret);
  899. }
  900. if (del_nr == 0) {
  901. fi = btrfs_item_ptr(leaf, path->slots[0],
  902. struct btrfs_file_extent_item);
  903. btrfs_set_file_extent_type(leaf, fi,
  904. BTRFS_FILE_EXTENT_REG);
  905. btrfs_mark_buffer_dirty(leaf);
  906. } else {
  907. fi = btrfs_item_ptr(leaf, del_slot - 1,
  908. struct btrfs_file_extent_item);
  909. btrfs_set_file_extent_type(leaf, fi,
  910. BTRFS_FILE_EXTENT_REG);
  911. btrfs_set_file_extent_num_bytes(leaf, fi,
  912. extent_end - key.offset);
  913. btrfs_mark_buffer_dirty(leaf);
  914. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  915. BUG_ON(ret);
  916. }
  917. out:
  918. btrfs_free_path(path);
  919. return 0;
  920. }
  921. /*
  922. * on error we return an unlocked page and the error value
  923. * on success we return a locked page and 0
  924. */
  925. static int prepare_uptodate_page(struct page *page, u64 pos,
  926. bool force_uptodate)
  927. {
  928. int ret = 0;
  929. if (((pos & (PAGE_CACHE_SIZE - 1)) || force_uptodate) &&
  930. !PageUptodate(page)) {
  931. ret = btrfs_readpage(NULL, page);
  932. if (ret)
  933. return ret;
  934. lock_page(page);
  935. if (!PageUptodate(page)) {
  936. unlock_page(page);
  937. return -EIO;
  938. }
  939. }
  940. return 0;
  941. }
  942. /*
  943. * this gets pages into the page cache and locks them down, it also properly
  944. * waits for data=ordered extents to finish before allowing the pages to be
  945. * modified.
  946. */
  947. static noinline int prepare_pages(struct btrfs_root *root, struct file *file,
  948. struct page **pages, size_t num_pages,
  949. loff_t pos, unsigned long first_index,
  950. size_t write_bytes, bool force_uptodate)
  951. {
  952. struct extent_state *cached_state = NULL;
  953. int i;
  954. unsigned long index = pos >> PAGE_CACHE_SHIFT;
  955. struct inode *inode = fdentry(file)->d_inode;
  956. gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
  957. int err = 0;
  958. int faili = 0;
  959. u64 start_pos;
  960. u64 last_pos;
  961. start_pos = pos & ~((u64)root->sectorsize - 1);
  962. last_pos = ((u64)index + num_pages) << PAGE_CACHE_SHIFT;
  963. again:
  964. for (i = 0; i < num_pages; i++) {
  965. pages[i] = find_or_create_page(inode->i_mapping, index + i,
  966. mask);
  967. if (!pages[i]) {
  968. faili = i - 1;
  969. err = -ENOMEM;
  970. goto fail;
  971. }
  972. if (i == 0)
  973. err = prepare_uptodate_page(pages[i], pos,
  974. force_uptodate);
  975. if (i == num_pages - 1)
  976. err = prepare_uptodate_page(pages[i],
  977. pos + write_bytes, false);
  978. if (err) {
  979. page_cache_release(pages[i]);
  980. faili = i - 1;
  981. goto fail;
  982. }
  983. wait_on_page_writeback(pages[i]);
  984. }
  985. err = 0;
  986. if (start_pos < inode->i_size) {
  987. struct btrfs_ordered_extent *ordered;
  988. lock_extent_bits(&BTRFS_I(inode)->io_tree,
  989. start_pos, last_pos - 1, 0, &cached_state,
  990. GFP_NOFS);
  991. ordered = btrfs_lookup_first_ordered_extent(inode,
  992. last_pos - 1);
  993. if (ordered &&
  994. ordered->file_offset + ordered->len > start_pos &&
  995. ordered->file_offset < last_pos) {
  996. btrfs_put_ordered_extent(ordered);
  997. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  998. start_pos, last_pos - 1,
  999. &cached_state, GFP_NOFS);
  1000. for (i = 0; i < num_pages; i++) {
  1001. unlock_page(pages[i]);
  1002. page_cache_release(pages[i]);
  1003. }
  1004. btrfs_wait_ordered_range(inode, start_pos,
  1005. last_pos - start_pos);
  1006. goto again;
  1007. }
  1008. if (ordered)
  1009. btrfs_put_ordered_extent(ordered);
  1010. clear_extent_bit(&BTRFS_I(inode)->io_tree, start_pos,
  1011. last_pos - 1, EXTENT_DIRTY | EXTENT_DELALLOC |
  1012. EXTENT_DO_ACCOUNTING, 0, 0, &cached_state,
  1013. GFP_NOFS);
  1014. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1015. start_pos, last_pos - 1, &cached_state,
  1016. GFP_NOFS);
  1017. }
  1018. for (i = 0; i < num_pages; i++) {
  1019. clear_page_dirty_for_io(pages[i]);
  1020. set_page_extent_mapped(pages[i]);
  1021. WARN_ON(!PageLocked(pages[i]));
  1022. }
  1023. return 0;
  1024. fail:
  1025. while (faili >= 0) {
  1026. unlock_page(pages[faili]);
  1027. page_cache_release(pages[faili]);
  1028. faili--;
  1029. }
  1030. return err;
  1031. }
  1032. static noinline ssize_t __btrfs_buffered_write(struct file *file,
  1033. struct iov_iter *i,
  1034. loff_t pos)
  1035. {
  1036. struct inode *inode = fdentry(file)->d_inode;
  1037. struct btrfs_root *root = BTRFS_I(inode)->root;
  1038. struct page **pages = NULL;
  1039. unsigned long first_index;
  1040. size_t num_written = 0;
  1041. int nrptrs;
  1042. int ret = 0;
  1043. bool force_page_uptodate = false;
  1044. nrptrs = min((iov_iter_count(i) + PAGE_CACHE_SIZE - 1) /
  1045. PAGE_CACHE_SIZE, PAGE_CACHE_SIZE /
  1046. (sizeof(struct page *)));
  1047. pages = kmalloc(nrptrs * sizeof(struct page *), GFP_KERNEL);
  1048. if (!pages)
  1049. return -ENOMEM;
  1050. first_index = pos >> PAGE_CACHE_SHIFT;
  1051. while (iov_iter_count(i) > 0) {
  1052. size_t offset = pos & (PAGE_CACHE_SIZE - 1);
  1053. size_t write_bytes = min(iov_iter_count(i),
  1054. nrptrs * (size_t)PAGE_CACHE_SIZE -
  1055. offset);
  1056. size_t num_pages = (write_bytes + offset +
  1057. PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1058. size_t dirty_pages;
  1059. size_t copied;
  1060. WARN_ON(num_pages > nrptrs);
  1061. /*
  1062. * Fault pages before locking them in prepare_pages
  1063. * to avoid recursive lock
  1064. */
  1065. if (unlikely(iov_iter_fault_in_readable(i, write_bytes))) {
  1066. ret = -EFAULT;
  1067. break;
  1068. }
  1069. ret = btrfs_delalloc_reserve_space(inode,
  1070. num_pages << PAGE_CACHE_SHIFT);
  1071. if (ret)
  1072. break;
  1073. /*
  1074. * This is going to setup the pages array with the number of
  1075. * pages we want, so we don't really need to worry about the
  1076. * contents of pages from loop to loop
  1077. */
  1078. ret = prepare_pages(root, file, pages, num_pages,
  1079. pos, first_index, write_bytes,
  1080. force_page_uptodate);
  1081. if (ret) {
  1082. btrfs_delalloc_release_space(inode,
  1083. num_pages << PAGE_CACHE_SHIFT);
  1084. break;
  1085. }
  1086. copied = btrfs_copy_from_user(pos, num_pages,
  1087. write_bytes, pages, i);
  1088. /*
  1089. * if we have trouble faulting in the pages, fall
  1090. * back to one page at a time
  1091. */
  1092. if (copied < write_bytes)
  1093. nrptrs = 1;
  1094. if (copied == 0) {
  1095. force_page_uptodate = true;
  1096. dirty_pages = 0;
  1097. } else {
  1098. force_page_uptodate = false;
  1099. dirty_pages = (copied + offset +
  1100. PAGE_CACHE_SIZE - 1) >>
  1101. PAGE_CACHE_SHIFT;
  1102. }
  1103. /*
  1104. * If we had a short copy we need to release the excess delaloc
  1105. * bytes we reserved. We need to increment outstanding_extents
  1106. * because btrfs_delalloc_release_space will decrement it, but
  1107. * we still have an outstanding extent for the chunk we actually
  1108. * managed to copy.
  1109. */
  1110. if (num_pages > dirty_pages) {
  1111. if (copied > 0) {
  1112. spin_lock(&BTRFS_I(inode)->lock);
  1113. BTRFS_I(inode)->outstanding_extents++;
  1114. spin_unlock(&BTRFS_I(inode)->lock);
  1115. }
  1116. btrfs_delalloc_release_space(inode,
  1117. (num_pages - dirty_pages) <<
  1118. PAGE_CACHE_SHIFT);
  1119. }
  1120. if (copied > 0) {
  1121. ret = btrfs_dirty_pages(root, inode, pages,
  1122. dirty_pages, pos, copied,
  1123. NULL);
  1124. if (ret) {
  1125. btrfs_delalloc_release_space(inode,
  1126. dirty_pages << PAGE_CACHE_SHIFT);
  1127. btrfs_drop_pages(pages, num_pages);
  1128. break;
  1129. }
  1130. }
  1131. btrfs_drop_pages(pages, num_pages);
  1132. cond_resched();
  1133. balance_dirty_pages_ratelimited_nr(inode->i_mapping,
  1134. dirty_pages);
  1135. if (dirty_pages < (root->leafsize >> PAGE_CACHE_SHIFT) + 1)
  1136. btrfs_btree_balance_dirty(root, 1);
  1137. btrfs_throttle(root);
  1138. pos += copied;
  1139. num_written += copied;
  1140. }
  1141. kfree(pages);
  1142. return num_written ? num_written : ret;
  1143. }
  1144. static ssize_t __btrfs_direct_write(struct kiocb *iocb,
  1145. const struct iovec *iov,
  1146. unsigned long nr_segs, loff_t pos,
  1147. loff_t *ppos, size_t count, size_t ocount)
  1148. {
  1149. struct file *file = iocb->ki_filp;
  1150. struct inode *inode = fdentry(file)->d_inode;
  1151. struct iov_iter i;
  1152. ssize_t written;
  1153. ssize_t written_buffered;
  1154. loff_t endbyte;
  1155. int err;
  1156. written = generic_file_direct_write(iocb, iov, &nr_segs, pos, ppos,
  1157. count, ocount);
  1158. /*
  1159. * the generic O_DIRECT will update in-memory i_size after the
  1160. * DIOs are done. But our endio handlers that update the on
  1161. * disk i_size never update past the in memory i_size. So we
  1162. * need one more update here to catch any additions to the
  1163. * file
  1164. */
  1165. if (inode->i_size != BTRFS_I(inode)->disk_i_size) {
  1166. btrfs_ordered_update_i_size(inode, inode->i_size, NULL);
  1167. mark_inode_dirty(inode);
  1168. }
  1169. if (written < 0 || written == count)
  1170. return written;
  1171. pos += written;
  1172. count -= written;
  1173. iov_iter_init(&i, iov, nr_segs, count, written);
  1174. written_buffered = __btrfs_buffered_write(file, &i, pos);
  1175. if (written_buffered < 0) {
  1176. err = written_buffered;
  1177. goto out;
  1178. }
  1179. endbyte = pos + written_buffered - 1;
  1180. err = filemap_write_and_wait_range(file->f_mapping, pos, endbyte);
  1181. if (err)
  1182. goto out;
  1183. written += written_buffered;
  1184. *ppos = pos + written_buffered;
  1185. invalidate_mapping_pages(file->f_mapping, pos >> PAGE_CACHE_SHIFT,
  1186. endbyte >> PAGE_CACHE_SHIFT);
  1187. out:
  1188. return written ? written : err;
  1189. }
  1190. static ssize_t btrfs_file_aio_write(struct kiocb *iocb,
  1191. const struct iovec *iov,
  1192. unsigned long nr_segs, loff_t pos)
  1193. {
  1194. struct file *file = iocb->ki_filp;
  1195. struct inode *inode = fdentry(file)->d_inode;
  1196. struct btrfs_root *root = BTRFS_I(inode)->root;
  1197. loff_t *ppos = &iocb->ki_pos;
  1198. u64 start_pos;
  1199. ssize_t num_written = 0;
  1200. ssize_t err = 0;
  1201. size_t count, ocount;
  1202. vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);
  1203. mutex_lock(&inode->i_mutex);
  1204. err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
  1205. if (err) {
  1206. mutex_unlock(&inode->i_mutex);
  1207. goto out;
  1208. }
  1209. count = ocount;
  1210. current->backing_dev_info = inode->i_mapping->backing_dev_info;
  1211. err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
  1212. if (err) {
  1213. mutex_unlock(&inode->i_mutex);
  1214. goto out;
  1215. }
  1216. if (count == 0) {
  1217. mutex_unlock(&inode->i_mutex);
  1218. goto out;
  1219. }
  1220. err = file_remove_suid(file);
  1221. if (err) {
  1222. mutex_unlock(&inode->i_mutex);
  1223. goto out;
  1224. }
  1225. /*
  1226. * If BTRFS flips readonly due to some impossible error
  1227. * (fs_info->fs_state now has BTRFS_SUPER_FLAG_ERROR),
  1228. * although we have opened a file as writable, we have
  1229. * to stop this write operation to ensure FS consistency.
  1230. */
  1231. if (root->fs_info->fs_state & BTRFS_SUPER_FLAG_ERROR) {
  1232. mutex_unlock(&inode->i_mutex);
  1233. err = -EROFS;
  1234. goto out;
  1235. }
  1236. file_update_time(file);
  1237. BTRFS_I(inode)->sequence++;
  1238. start_pos = round_down(pos, root->sectorsize);
  1239. if (start_pos > i_size_read(inode)) {
  1240. err = btrfs_cont_expand(inode, i_size_read(inode), start_pos);
  1241. if (err) {
  1242. mutex_unlock(&inode->i_mutex);
  1243. goto out;
  1244. }
  1245. }
  1246. if (unlikely(file->f_flags & O_DIRECT)) {
  1247. num_written = __btrfs_direct_write(iocb, iov, nr_segs,
  1248. pos, ppos, count, ocount);
  1249. } else {
  1250. struct iov_iter i;
  1251. iov_iter_init(&i, iov, nr_segs, count, num_written);
  1252. num_written = __btrfs_buffered_write(file, &i, pos);
  1253. if (num_written > 0)
  1254. *ppos = pos + num_written;
  1255. }
  1256. mutex_unlock(&inode->i_mutex);
  1257. /*
  1258. * we want to make sure fsync finds this change
  1259. * but we haven't joined a transaction running right now.
  1260. *
  1261. * Later on, someone is sure to update the inode and get the
  1262. * real transid recorded.
  1263. *
  1264. * We set last_trans now to the fs_info generation + 1,
  1265. * this will either be one more than the running transaction
  1266. * or the generation used for the next transaction if there isn't
  1267. * one running right now.
  1268. */
  1269. BTRFS_I(inode)->last_trans = root->fs_info->generation + 1;
  1270. if (num_written > 0 || num_written == -EIOCBQUEUED) {
  1271. err = generic_write_sync(file, pos, num_written);
  1272. if (err < 0 && num_written > 0)
  1273. num_written = err;
  1274. }
  1275. out:
  1276. current->backing_dev_info = NULL;
  1277. return num_written ? num_written : err;
  1278. }
  1279. int btrfs_release_file(struct inode *inode, struct file *filp)
  1280. {
  1281. /*
  1282. * ordered_data_close is set by settattr when we are about to truncate
  1283. * a file from a non-zero size to a zero size. This tries to
  1284. * flush down new bytes that may have been written if the
  1285. * application were using truncate to replace a file in place.
  1286. */
  1287. if (BTRFS_I(inode)->ordered_data_close) {
  1288. BTRFS_I(inode)->ordered_data_close = 0;
  1289. btrfs_add_ordered_operation(NULL, BTRFS_I(inode)->root, inode);
  1290. if (inode->i_size > BTRFS_ORDERED_OPERATIONS_FLUSH_LIMIT)
  1291. filemap_flush(inode->i_mapping);
  1292. }
  1293. if (filp->private_data)
  1294. btrfs_ioctl_trans_end(filp);
  1295. return 0;
  1296. }
  1297. /*
  1298. * fsync call for both files and directories. This logs the inode into
  1299. * the tree log instead of forcing full commits whenever possible.
  1300. *
  1301. * It needs to call filemap_fdatawait so that all ordered extent updates are
  1302. * in the metadata btree are up to date for copying to the log.
  1303. *
  1304. * It drops the inode mutex before doing the tree log commit. This is an
  1305. * important optimization for directories because holding the mutex prevents
  1306. * new operations on the dir while we write to disk.
  1307. */
  1308. int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
  1309. {
  1310. struct dentry *dentry = file->f_path.dentry;
  1311. struct inode *inode = dentry->d_inode;
  1312. struct btrfs_root *root = BTRFS_I(inode)->root;
  1313. int ret = 0;
  1314. struct btrfs_trans_handle *trans;
  1315. trace_btrfs_sync_file(file, datasync);
  1316. ret = filemap_write_and_wait_range(inode->i_mapping, start, end);
  1317. if (ret)
  1318. return ret;
  1319. mutex_lock(&inode->i_mutex);
  1320. /* we wait first, since the writeback may change the inode */
  1321. root->log_batch++;
  1322. btrfs_wait_ordered_range(inode, 0, (u64)-1);
  1323. root->log_batch++;
  1324. /*
  1325. * check the transaction that last modified this inode
  1326. * and see if its already been committed
  1327. */
  1328. if (!BTRFS_I(inode)->last_trans) {
  1329. mutex_unlock(&inode->i_mutex);
  1330. goto out;
  1331. }
  1332. /*
  1333. * if the last transaction that changed this file was before
  1334. * the current transaction, we can bail out now without any
  1335. * syncing
  1336. */
  1337. smp_mb();
  1338. if (BTRFS_I(inode)->last_trans <=
  1339. root->fs_info->last_trans_committed) {
  1340. BTRFS_I(inode)->last_trans = 0;
  1341. mutex_unlock(&inode->i_mutex);
  1342. goto out;
  1343. }
  1344. /*
  1345. * ok we haven't committed the transaction yet, lets do a commit
  1346. */
  1347. if (file->private_data)
  1348. btrfs_ioctl_trans_end(file);
  1349. trans = btrfs_start_transaction(root, 0);
  1350. if (IS_ERR(trans)) {
  1351. ret = PTR_ERR(trans);
  1352. mutex_unlock(&inode->i_mutex);
  1353. goto out;
  1354. }
  1355. ret = btrfs_log_dentry_safe(trans, root, dentry);
  1356. if (ret < 0) {
  1357. mutex_unlock(&inode->i_mutex);
  1358. goto out;
  1359. }
  1360. /* we've logged all the items and now have a consistent
  1361. * version of the file in the log. It is possible that
  1362. * someone will come in and modify the file, but that's
  1363. * fine because the log is consistent on disk, and we
  1364. * have references to all of the file's extents
  1365. *
  1366. * It is possible that someone will come in and log the
  1367. * file again, but that will end up using the synchronization
  1368. * inside btrfs_sync_log to keep things safe.
  1369. */
  1370. mutex_unlock(&inode->i_mutex);
  1371. if (ret != BTRFS_NO_LOG_SYNC) {
  1372. if (ret > 0) {
  1373. ret = btrfs_commit_transaction(trans, root);
  1374. } else {
  1375. ret = btrfs_sync_log(trans, root);
  1376. if (ret == 0)
  1377. ret = btrfs_end_transaction(trans, root);
  1378. else
  1379. ret = btrfs_commit_transaction(trans, root);
  1380. }
  1381. } else {
  1382. ret = btrfs_end_transaction(trans, root);
  1383. }
  1384. out:
  1385. return ret > 0 ? -EIO : ret;
  1386. }
  1387. static const struct vm_operations_struct btrfs_file_vm_ops = {
  1388. .fault = filemap_fault,
  1389. .page_mkwrite = btrfs_page_mkwrite,
  1390. };
  1391. static int btrfs_file_mmap(struct file *filp, struct vm_area_struct *vma)
  1392. {
  1393. struct address_space *mapping = filp->f_mapping;
  1394. if (!mapping->a_ops->readpage)
  1395. return -ENOEXEC;
  1396. file_accessed(filp);
  1397. vma->vm_ops = &btrfs_file_vm_ops;
  1398. vma->vm_flags |= VM_CAN_NONLINEAR;
  1399. return 0;
  1400. }
  1401. static long btrfs_fallocate(struct file *file, int mode,
  1402. loff_t offset, loff_t len)
  1403. {
  1404. struct inode *inode = file->f_path.dentry->d_inode;
  1405. struct extent_state *cached_state = NULL;
  1406. u64 cur_offset;
  1407. u64 last_byte;
  1408. u64 alloc_start;
  1409. u64 alloc_end;
  1410. u64 alloc_hint = 0;
  1411. u64 locked_end;
  1412. u64 mask = BTRFS_I(inode)->root->sectorsize - 1;
  1413. struct extent_map *em;
  1414. int ret;
  1415. alloc_start = offset & ~mask;
  1416. alloc_end = (offset + len + mask) & ~mask;
  1417. /* We only support the FALLOC_FL_KEEP_SIZE mode */
  1418. if (mode & ~FALLOC_FL_KEEP_SIZE)
  1419. return -EOPNOTSUPP;
  1420. /*
  1421. * wait for ordered IO before we have any locks. We'll loop again
  1422. * below with the locks held.
  1423. */
  1424. btrfs_wait_ordered_range(inode, alloc_start, alloc_end - alloc_start);
  1425. mutex_lock(&inode->i_mutex);
  1426. ret = inode_newsize_ok(inode, alloc_end);
  1427. if (ret)
  1428. goto out;
  1429. if (alloc_start > inode->i_size) {
  1430. ret = btrfs_cont_expand(inode, i_size_read(inode),
  1431. alloc_start);
  1432. if (ret)
  1433. goto out;
  1434. }
  1435. locked_end = alloc_end - 1;
  1436. while (1) {
  1437. struct btrfs_ordered_extent *ordered;
  1438. /* the extent lock is ordered inside the running
  1439. * transaction
  1440. */
  1441. lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
  1442. locked_end, 0, &cached_state, GFP_NOFS);
  1443. ordered = btrfs_lookup_first_ordered_extent(inode,
  1444. alloc_end - 1);
  1445. if (ordered &&
  1446. ordered->file_offset + ordered->len > alloc_start &&
  1447. ordered->file_offset < alloc_end) {
  1448. btrfs_put_ordered_extent(ordered);
  1449. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1450. alloc_start, locked_end,
  1451. &cached_state, GFP_NOFS);
  1452. /*
  1453. * we can't wait on the range with the transaction
  1454. * running or with the extent lock held
  1455. */
  1456. btrfs_wait_ordered_range(inode, alloc_start,
  1457. alloc_end - alloc_start);
  1458. } else {
  1459. if (ordered)
  1460. btrfs_put_ordered_extent(ordered);
  1461. break;
  1462. }
  1463. }
  1464. cur_offset = alloc_start;
  1465. while (1) {
  1466. u64 actual_end;
  1467. em = btrfs_get_extent(inode, NULL, 0, cur_offset,
  1468. alloc_end - cur_offset, 0);
  1469. BUG_ON(IS_ERR_OR_NULL(em));
  1470. last_byte = min(extent_map_end(em), alloc_end);
  1471. actual_end = min_t(u64, extent_map_end(em), offset + len);
  1472. last_byte = (last_byte + mask) & ~mask;
  1473. if (em->block_start == EXTENT_MAP_HOLE ||
  1474. (cur_offset >= inode->i_size &&
  1475. !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
  1476. /*
  1477. * Make sure we have enough space before we do the
  1478. * allocation.
  1479. */
  1480. ret = btrfs_check_data_free_space(inode, last_byte -
  1481. cur_offset);
  1482. if (ret) {
  1483. free_extent_map(em);
  1484. break;
  1485. }
  1486. ret = btrfs_prealloc_file_range(inode, mode, cur_offset,
  1487. last_byte - cur_offset,
  1488. 1 << inode->i_blkbits,
  1489. offset + len,
  1490. &alloc_hint);
  1491. /* Let go of our reservation. */
  1492. btrfs_free_reserved_data_space(inode, last_byte -
  1493. cur_offset);
  1494. if (ret < 0) {
  1495. free_extent_map(em);
  1496. break;
  1497. }
  1498. } else if (actual_end > inode->i_size &&
  1499. !(mode & FALLOC_FL_KEEP_SIZE)) {
  1500. /*
  1501. * We didn't need to allocate any more space, but we
  1502. * still extended the size of the file so we need to
  1503. * update i_size.
  1504. */
  1505. inode->i_ctime = CURRENT_TIME;
  1506. i_size_write(inode, actual_end);
  1507. btrfs_ordered_update_i_size(inode, actual_end, NULL);
  1508. }
  1509. free_extent_map(em);
  1510. cur_offset = last_byte;
  1511. if (cur_offset >= alloc_end) {
  1512. ret = 0;
  1513. break;
  1514. }
  1515. }
  1516. unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
  1517. &cached_state, GFP_NOFS);
  1518. out:
  1519. mutex_unlock(&inode->i_mutex);
  1520. return ret;
  1521. }
  1522. static int find_desired_extent(struct inode *inode, loff_t *offset, int origin)
  1523. {
  1524. struct btrfs_root *root = BTRFS_I(inode)->root;
  1525. struct extent_map *em;
  1526. struct extent_state *cached_state = NULL;
  1527. u64 lockstart = *offset;
  1528. u64 lockend = i_size_read(inode);
  1529. u64 start = *offset;
  1530. u64 orig_start = *offset;
  1531. u64 len = i_size_read(inode);
  1532. u64 last_end = 0;
  1533. int ret = 0;
  1534. lockend = max_t(u64, root->sectorsize, lockend);
  1535. if (lockend <= lockstart)
  1536. lockend = lockstart + root->sectorsize;
  1537. len = lockend - lockstart + 1;
  1538. len = max_t(u64, len, root->sectorsize);
  1539. if (inode->i_size == 0)
  1540. return -ENXIO;
  1541. lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, 0,
  1542. &cached_state, GFP_NOFS);
  1543. /*
  1544. * Delalloc is such a pain. If we have a hole and we have pending
  1545. * delalloc for a portion of the hole we will get back a hole that
  1546. * exists for the entire range since it hasn't been actually written
  1547. * yet. So to take care of this case we need to look for an extent just
  1548. * before the position we want in case there is outstanding delalloc
  1549. * going on here.
  1550. */
  1551. if (origin == SEEK_HOLE && start != 0) {
  1552. if (start <= root->sectorsize)
  1553. em = btrfs_get_extent_fiemap(inode, NULL, 0, 0,
  1554. root->sectorsize, 0);
  1555. else
  1556. em = btrfs_get_extent_fiemap(inode, NULL, 0,
  1557. start - root->sectorsize,
  1558. root->sectorsize, 0);
  1559. if (IS_ERR(em)) {
  1560. ret = -ENXIO;
  1561. goto out;
  1562. }
  1563. last_end = em->start + em->len;
  1564. if (em->block_start == EXTENT_MAP_DELALLOC)
  1565. last_end = min_t(u64, last_end, inode->i_size);
  1566. free_extent_map(em);
  1567. }
  1568. while (1) {
  1569. em = btrfs_get_extent_fiemap(inode, NULL, 0, start, len, 0);
  1570. if (IS_ERR(em)) {
  1571. ret = -ENXIO;
  1572. break;
  1573. }
  1574. if (em->block_start == EXTENT_MAP_HOLE) {
  1575. if (test_bit(EXTENT_FLAG_VACANCY, &em->flags)) {
  1576. if (last_end <= orig_start) {
  1577. free_extent_map(em);
  1578. ret = -ENXIO;
  1579. break;
  1580. }
  1581. }
  1582. if (origin == SEEK_HOLE) {
  1583. *offset = start;
  1584. free_extent_map(em);
  1585. break;
  1586. }
  1587. } else {
  1588. if (origin == SEEK_DATA) {
  1589. if (em->block_start == EXTENT_MAP_DELALLOC) {
  1590. if (start >= inode->i_size) {
  1591. free_extent_map(em);
  1592. ret = -ENXIO;
  1593. break;
  1594. }
  1595. }
  1596. *offset = start;
  1597. free_extent_map(em);
  1598. break;
  1599. }
  1600. }
  1601. start = em->start + em->len;
  1602. last_end = em->start + em->len;
  1603. if (em->block_start == EXTENT_MAP_DELALLOC)
  1604. last_end = min_t(u64, last_end, inode->i_size);
  1605. if (test_bit(EXTENT_FLAG_VACANCY, &em->flags)) {
  1606. free_extent_map(em);
  1607. ret = -ENXIO;
  1608. break;
  1609. }
  1610. free_extent_map(em);
  1611. cond_resched();
  1612. }
  1613. if (!ret)
  1614. *offset = min(*offset, inode->i_size);
  1615. out:
  1616. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  1617. &cached_state, GFP_NOFS);
  1618. return ret;
  1619. }
  1620. static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int origin)
  1621. {
  1622. struct inode *inode = file->f_mapping->host;
  1623. int ret;
  1624. mutex_lock(&inode->i_mutex);
  1625. switch (origin) {
  1626. case SEEK_END:
  1627. case SEEK_CUR:
  1628. offset = generic_file_llseek(file, offset, origin);
  1629. goto out;
  1630. case SEEK_DATA:
  1631. case SEEK_HOLE:
  1632. if (offset >= i_size_read(inode)) {
  1633. mutex_unlock(&inode->i_mutex);
  1634. return -ENXIO;
  1635. }
  1636. ret = find_desired_extent(inode, &offset, origin);
  1637. if (ret) {
  1638. mutex_unlock(&inode->i_mutex);
  1639. return ret;
  1640. }
  1641. }
  1642. if (offset < 0 && !(file->f_mode & FMODE_UNSIGNED_OFFSET)) {
  1643. offset = -EINVAL;
  1644. goto out;
  1645. }
  1646. if (offset > inode->i_sb->s_maxbytes) {
  1647. offset = -EINVAL;
  1648. goto out;
  1649. }
  1650. /* Special lock needed here? */
  1651. if (offset != file->f_pos) {
  1652. file->f_pos = offset;
  1653. file->f_version = 0;
  1654. }
  1655. out:
  1656. mutex_unlock(&inode->i_mutex);
  1657. return offset;
  1658. }
  1659. const struct file_operations btrfs_file_operations = {
  1660. .llseek = btrfs_file_llseek,
  1661. .read = do_sync_read,
  1662. .write = do_sync_write,
  1663. .aio_read = generic_file_aio_read,
  1664. .splice_read = generic_file_splice_read,
  1665. .aio_write = btrfs_file_aio_write,
  1666. .mmap = btrfs_file_mmap,
  1667. .open = generic_file_open,
  1668. .release = btrfs_release_file,
  1669. .fsync = btrfs_sync_file,
  1670. .fallocate = btrfs_fallocate,
  1671. .unlocked_ioctl = btrfs_ioctl,
  1672. #ifdef CONFIG_COMPAT
  1673. .compat_ioctl = btrfs_ioctl,
  1674. #endif
  1675. };