move_extent.c 44 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528
  1. /*
  2. * Copyright (c) 2008,2009 NEC Software Tohoku, Ltd.
  3. * Written by Takashi Sato <t-sato@yk.jp.nec.com>
  4. * Akira Fujita <a-fujita@rs.jp.nec.com>
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of version 2.1 of the GNU Lesser General Public License
  8. * as published by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. */
  15. #include <linux/fs.h>
  16. #include <linux/quotaops.h>
  17. #include <linux/slab.h>
  18. #include "ext4_jbd2.h"
  19. #include "ext4.h"
  20. /**
  21. * get_ext_path - Find an extent path for designated logical block number.
  22. *
  23. * @inode: an inode which is searched
  24. * @lblock: logical block number to find an extent path
  25. * @path: pointer to an extent path pointer (for output)
  26. *
  27. * ext4_ext_find_extent wrapper. Return 0 on success, or a negative error value
  28. * on failure.
  29. */
  30. static inline int
  31. get_ext_path(struct inode *inode, ext4_lblk_t lblock,
  32. struct ext4_ext_path **path)
  33. {
  34. int ret = 0;
  35. *path = ext4_ext_find_extent(inode, lblock, *path);
  36. if (IS_ERR(*path)) {
  37. ret = PTR_ERR(*path);
  38. *path = NULL;
  39. } else if ((*path)[ext_depth(inode)].p_ext == NULL)
  40. ret = -ENODATA;
  41. return ret;
  42. }
  43. /**
  44. * copy_extent_status - Copy the extent's initialization status
  45. *
  46. * @src: an extent for getting initialize status
  47. * @dest: an extent to be set the status
  48. */
  49. static void
  50. copy_extent_status(struct ext4_extent *src, struct ext4_extent *dest)
  51. {
  52. if (ext4_ext_is_uninitialized(src))
  53. ext4_ext_mark_uninitialized(dest);
  54. else
  55. dest->ee_len = cpu_to_le16(ext4_ext_get_actual_len(dest));
  56. }
  57. /**
  58. * mext_next_extent - Search for the next extent and set it to "extent"
  59. *
  60. * @inode: inode which is searched
  61. * @path: this will obtain data for the next extent
  62. * @extent: pointer to the next extent we have just gotten
  63. *
  64. * Search the next extent in the array of ext4_ext_path structure (@path)
  65. * and set it to ext4_extent structure (@extent). In addition, the member of
  66. * @path (->p_ext) also points the next extent. Return 0 on success, 1 if
  67. * ext4_ext_path structure refers to the last extent, or a negative error
  68. * value on failure.
  69. */
  70. static int
  71. mext_next_extent(struct inode *inode, struct ext4_ext_path *path,
  72. struct ext4_extent **extent)
  73. {
  74. struct ext4_extent_header *eh;
  75. int ppos, leaf_ppos = path->p_depth;
  76. ppos = leaf_ppos;
  77. if (EXT_LAST_EXTENT(path[ppos].p_hdr) > path[ppos].p_ext) {
  78. /* leaf block */
  79. *extent = ++path[ppos].p_ext;
  80. path[ppos].p_block = ext4_ext_pblock(path[ppos].p_ext);
  81. return 0;
  82. }
  83. while (--ppos >= 0) {
  84. if (EXT_LAST_INDEX(path[ppos].p_hdr) >
  85. path[ppos].p_idx) {
  86. int cur_ppos = ppos;
  87. /* index block */
  88. path[ppos].p_idx++;
  89. path[ppos].p_block = ext4_idx_pblock(path[ppos].p_idx);
  90. if (path[ppos+1].p_bh)
  91. brelse(path[ppos+1].p_bh);
  92. path[ppos+1].p_bh =
  93. sb_bread(inode->i_sb, path[ppos].p_block);
  94. if (!path[ppos+1].p_bh)
  95. return -EIO;
  96. path[ppos+1].p_hdr =
  97. ext_block_hdr(path[ppos+1].p_bh);
  98. /* Halfway index block */
  99. while (++cur_ppos < leaf_ppos) {
  100. path[cur_ppos].p_idx =
  101. EXT_FIRST_INDEX(path[cur_ppos].p_hdr);
  102. path[cur_ppos].p_block =
  103. ext4_idx_pblock(path[cur_ppos].p_idx);
  104. if (path[cur_ppos+1].p_bh)
  105. brelse(path[cur_ppos+1].p_bh);
  106. path[cur_ppos+1].p_bh = sb_bread(inode->i_sb,
  107. path[cur_ppos].p_block);
  108. if (!path[cur_ppos+1].p_bh)
  109. return -EIO;
  110. path[cur_ppos+1].p_hdr =
  111. ext_block_hdr(path[cur_ppos+1].p_bh);
  112. }
  113. path[leaf_ppos].p_ext = *extent = NULL;
  114. eh = path[leaf_ppos].p_hdr;
  115. if (le16_to_cpu(eh->eh_entries) == 0)
  116. /* empty leaf is found */
  117. return -ENODATA;
  118. /* leaf block */
  119. path[leaf_ppos].p_ext = *extent =
  120. EXT_FIRST_EXTENT(path[leaf_ppos].p_hdr);
  121. path[leaf_ppos].p_block =
  122. ext4_ext_pblock(path[leaf_ppos].p_ext);
  123. return 0;
  124. }
  125. }
  126. /* We found the last extent */
  127. return 1;
  128. }
  129. /**
  130. * double_down_write_data_sem - Acquire two inodes' write lock of i_data_sem
  131. *
  132. * Acquire write lock of i_data_sem of the two inodes
  133. */
  134. static void
  135. double_down_write_data_sem(struct inode *first, struct inode *second)
  136. {
  137. if (first < second) {
  138. down_write(&EXT4_I(first)->i_data_sem);
  139. down_write_nested(&EXT4_I(second)->i_data_sem, SINGLE_DEPTH_NESTING);
  140. } else {
  141. down_write(&EXT4_I(second)->i_data_sem);
  142. down_write_nested(&EXT4_I(first)->i_data_sem, SINGLE_DEPTH_NESTING);
  143. }
  144. }
  145. /**
  146. * double_up_write_data_sem - Release two inodes' write lock of i_data_sem
  147. *
  148. * @orig_inode: original inode structure to be released its lock first
  149. * @donor_inode: donor inode structure to be released its lock second
  150. * Release write lock of i_data_sem of two inodes (orig and donor).
  151. */
  152. static void
  153. double_up_write_data_sem(struct inode *orig_inode, struct inode *donor_inode)
  154. {
  155. up_write(&EXT4_I(orig_inode)->i_data_sem);
  156. up_write(&EXT4_I(donor_inode)->i_data_sem);
  157. }
  158. /**
  159. * mext_insert_across_blocks - Insert extents across leaf block
  160. *
  161. * @handle: journal handle
  162. * @orig_inode: original inode
  163. * @o_start: first original extent to be changed
  164. * @o_end: last original extent to be changed
  165. * @start_ext: first new extent to be inserted
  166. * @new_ext: middle of new extent to be inserted
  167. * @end_ext: last new extent to be inserted
  168. *
  169. * Allocate a new leaf block and insert extents into it. Return 0 on success,
  170. * or a negative error value on failure.
  171. */
  172. static int
  173. mext_insert_across_blocks(handle_t *handle, struct inode *orig_inode,
  174. struct ext4_extent *o_start, struct ext4_extent *o_end,
  175. struct ext4_extent *start_ext, struct ext4_extent *new_ext,
  176. struct ext4_extent *end_ext)
  177. {
  178. struct ext4_ext_path *orig_path = NULL;
  179. ext4_lblk_t eblock = 0;
  180. int new_flag = 0;
  181. int end_flag = 0;
  182. int err = 0;
  183. if (start_ext->ee_len && new_ext->ee_len && end_ext->ee_len) {
  184. if (o_start == o_end) {
  185. /* start_ext new_ext end_ext
  186. * donor |---------|-----------|--------|
  187. * orig |------------------------------|
  188. */
  189. end_flag = 1;
  190. } else {
  191. /* start_ext new_ext end_ext
  192. * donor |---------|----------|---------|
  193. * orig |---------------|--------------|
  194. */
  195. o_end->ee_block = end_ext->ee_block;
  196. o_end->ee_len = end_ext->ee_len;
  197. ext4_ext_store_pblock(o_end, ext4_ext_pblock(end_ext));
  198. }
  199. o_start->ee_len = start_ext->ee_len;
  200. eblock = le32_to_cpu(start_ext->ee_block);
  201. new_flag = 1;
  202. } else if (start_ext->ee_len && new_ext->ee_len &&
  203. !end_ext->ee_len && o_start == o_end) {
  204. /* start_ext new_ext
  205. * donor |--------------|---------------|
  206. * orig |------------------------------|
  207. */
  208. o_start->ee_len = start_ext->ee_len;
  209. eblock = le32_to_cpu(start_ext->ee_block);
  210. new_flag = 1;
  211. } else if (!start_ext->ee_len && new_ext->ee_len &&
  212. end_ext->ee_len && o_start == o_end) {
  213. /* new_ext end_ext
  214. * donor |--------------|---------------|
  215. * orig |------------------------------|
  216. */
  217. o_end->ee_block = end_ext->ee_block;
  218. o_end->ee_len = end_ext->ee_len;
  219. ext4_ext_store_pblock(o_end, ext4_ext_pblock(end_ext));
  220. /*
  221. * Set 0 to the extent block if new_ext was
  222. * the first block.
  223. */
  224. if (new_ext->ee_block)
  225. eblock = le32_to_cpu(new_ext->ee_block);
  226. new_flag = 1;
  227. } else {
  228. ext4_debug("ext4 move extent: Unexpected insert case\n");
  229. return -EIO;
  230. }
  231. if (new_flag) {
  232. err = get_ext_path(orig_inode, eblock, &orig_path);
  233. if (err)
  234. goto out;
  235. if (ext4_ext_insert_extent(handle, orig_inode,
  236. orig_path, new_ext, 0))
  237. goto out;
  238. }
  239. if (end_flag) {
  240. err = get_ext_path(orig_inode,
  241. le32_to_cpu(end_ext->ee_block) - 1, &orig_path);
  242. if (err)
  243. goto out;
  244. if (ext4_ext_insert_extent(handle, orig_inode,
  245. orig_path, end_ext, 0))
  246. goto out;
  247. }
  248. out:
  249. if (orig_path) {
  250. ext4_ext_drop_refs(orig_path);
  251. kfree(orig_path);
  252. }
  253. return err;
  254. }
  255. /**
  256. * mext_insert_inside_block - Insert new extent to the extent block
  257. *
  258. * @o_start: first original extent to be moved
  259. * @o_end: last original extent to be moved
  260. * @start_ext: first new extent to be inserted
  261. * @new_ext: middle of new extent to be inserted
  262. * @end_ext: last new extent to be inserted
  263. * @eh: extent header of target leaf block
  264. * @range_to_move: used to decide how to insert extent
  265. *
  266. * Insert extents into the leaf block. The extent (@o_start) is overwritten
  267. * by inserted extents.
  268. */
  269. static void
  270. mext_insert_inside_block(struct ext4_extent *o_start,
  271. struct ext4_extent *o_end,
  272. struct ext4_extent *start_ext,
  273. struct ext4_extent *new_ext,
  274. struct ext4_extent *end_ext,
  275. struct ext4_extent_header *eh,
  276. int range_to_move)
  277. {
  278. int i = 0;
  279. unsigned long len;
  280. /* Move the existing extents */
  281. if (range_to_move && o_end < EXT_LAST_EXTENT(eh)) {
  282. len = (unsigned long)(EXT_LAST_EXTENT(eh) + 1) -
  283. (unsigned long)(o_end + 1);
  284. memmove(o_end + 1 + range_to_move, o_end + 1, len);
  285. }
  286. /* Insert start entry */
  287. if (start_ext->ee_len)
  288. o_start[i++].ee_len = start_ext->ee_len;
  289. /* Insert new entry */
  290. if (new_ext->ee_len) {
  291. o_start[i] = *new_ext;
  292. ext4_ext_store_pblock(&o_start[i++], ext4_ext_pblock(new_ext));
  293. }
  294. /* Insert end entry */
  295. if (end_ext->ee_len)
  296. o_start[i] = *end_ext;
  297. /* Increment the total entries counter on the extent block */
  298. le16_add_cpu(&eh->eh_entries, range_to_move);
  299. }
  300. /**
  301. * mext_insert_extents - Insert new extent
  302. *
  303. * @handle: journal handle
  304. * @orig_inode: original inode
  305. * @orig_path: path indicates first extent to be changed
  306. * @o_start: first original extent to be changed
  307. * @o_end: last original extent to be changed
  308. * @start_ext: first new extent to be inserted
  309. * @new_ext: middle of new extent to be inserted
  310. * @end_ext: last new extent to be inserted
  311. *
  312. * Call the function to insert extents. If we cannot add more extents into
  313. * the leaf block, we call mext_insert_across_blocks() to create a
  314. * new leaf block. Otherwise call mext_insert_inside_block(). Return 0
  315. * on success, or a negative error value on failure.
  316. */
  317. static int
  318. mext_insert_extents(handle_t *handle, struct inode *orig_inode,
  319. struct ext4_ext_path *orig_path,
  320. struct ext4_extent *o_start,
  321. struct ext4_extent *o_end,
  322. struct ext4_extent *start_ext,
  323. struct ext4_extent *new_ext,
  324. struct ext4_extent *end_ext)
  325. {
  326. struct ext4_extent_header *eh;
  327. unsigned long need_slots, slots_range;
  328. int range_to_move, depth, ret;
  329. /*
  330. * The extents need to be inserted
  331. * start_extent + new_extent + end_extent.
  332. */
  333. need_slots = (start_ext->ee_len ? 1 : 0) + (end_ext->ee_len ? 1 : 0) +
  334. (new_ext->ee_len ? 1 : 0);
  335. /* The number of slots between start and end */
  336. slots_range = ((unsigned long)(o_end + 1) - (unsigned long)o_start + 1)
  337. / sizeof(struct ext4_extent);
  338. /* Range to move the end of extent */
  339. range_to_move = need_slots - slots_range;
  340. depth = orig_path->p_depth;
  341. orig_path += depth;
  342. eh = orig_path->p_hdr;
  343. if (depth) {
  344. /* Register to journal */
  345. ret = ext4_journal_get_write_access(handle, orig_path->p_bh);
  346. if (ret)
  347. return ret;
  348. }
  349. /* Expansion */
  350. if (range_to_move > 0 &&
  351. (range_to_move > le16_to_cpu(eh->eh_max)
  352. - le16_to_cpu(eh->eh_entries))) {
  353. ret = mext_insert_across_blocks(handle, orig_inode, o_start,
  354. o_end, start_ext, new_ext, end_ext);
  355. if (ret < 0)
  356. return ret;
  357. } else
  358. mext_insert_inside_block(o_start, o_end, start_ext, new_ext,
  359. end_ext, eh, range_to_move);
  360. if (depth) {
  361. ret = ext4_handle_dirty_metadata(handle, orig_inode,
  362. orig_path->p_bh);
  363. if (ret)
  364. return ret;
  365. } else {
  366. ret = ext4_mark_inode_dirty(handle, orig_inode);
  367. if (ret < 0)
  368. return ret;
  369. }
  370. return 0;
  371. }
  372. /**
  373. * mext_leaf_block - Move one leaf extent block into the inode.
  374. *
  375. * @handle: journal handle
  376. * @orig_inode: original inode
  377. * @orig_path: path indicates first extent to be changed
  378. * @dext: donor extent
  379. * @from: start offset on the target file
  380. *
  381. * In order to insert extents into the leaf block, we must divide the extent
  382. * in the leaf block into three extents. The one is located to be inserted
  383. * extents, and the others are located around it.
  384. *
  385. * Therefore, this function creates structures to save extents of the leaf
  386. * block, and inserts extents by calling mext_insert_extents() with
  387. * created extents. Return 0 on success, or a negative error value on failure.
  388. */
  389. static int
  390. mext_leaf_block(handle_t *handle, struct inode *orig_inode,
  391. struct ext4_ext_path *orig_path, struct ext4_extent *dext,
  392. ext4_lblk_t *from)
  393. {
  394. struct ext4_extent *oext, *o_start, *o_end, *prev_ext;
  395. struct ext4_extent new_ext, start_ext, end_ext;
  396. ext4_lblk_t new_ext_end;
  397. int oext_alen, new_ext_alen, end_ext_alen;
  398. int depth = ext_depth(orig_inode);
  399. int ret;
  400. start_ext.ee_block = end_ext.ee_block = 0;
  401. o_start = o_end = oext = orig_path[depth].p_ext;
  402. oext_alen = ext4_ext_get_actual_len(oext);
  403. start_ext.ee_len = end_ext.ee_len = 0;
  404. new_ext.ee_block = cpu_to_le32(*from);
  405. ext4_ext_store_pblock(&new_ext, ext4_ext_pblock(dext));
  406. new_ext.ee_len = dext->ee_len;
  407. new_ext_alen = ext4_ext_get_actual_len(&new_ext);
  408. new_ext_end = le32_to_cpu(new_ext.ee_block) + new_ext_alen - 1;
  409. /*
  410. * Case: original extent is first
  411. * oext |--------|
  412. * new_ext |--|
  413. * start_ext |--|
  414. */
  415. if (le32_to_cpu(oext->ee_block) < le32_to_cpu(new_ext.ee_block) &&
  416. le32_to_cpu(new_ext.ee_block) <
  417. le32_to_cpu(oext->ee_block) + oext_alen) {
  418. start_ext.ee_len = cpu_to_le16(le32_to_cpu(new_ext.ee_block) -
  419. le32_to_cpu(oext->ee_block));
  420. start_ext.ee_block = oext->ee_block;
  421. copy_extent_status(oext, &start_ext);
  422. } else if (oext > EXT_FIRST_EXTENT(orig_path[depth].p_hdr)) {
  423. prev_ext = oext - 1;
  424. /*
  425. * We can merge new_ext into previous extent,
  426. * if these are contiguous and same extent type.
  427. */
  428. if (ext4_can_extents_be_merged(orig_inode, prev_ext,
  429. &new_ext)) {
  430. o_start = prev_ext;
  431. start_ext.ee_len = cpu_to_le16(
  432. ext4_ext_get_actual_len(prev_ext) +
  433. new_ext_alen);
  434. start_ext.ee_block = oext->ee_block;
  435. copy_extent_status(prev_ext, &start_ext);
  436. new_ext.ee_len = 0;
  437. }
  438. }
  439. /*
  440. * Case: new_ext_end must be less than oext
  441. * oext |-----------|
  442. * new_ext |-------|
  443. */
  444. if (le32_to_cpu(oext->ee_block) + oext_alen - 1 < new_ext_end) {
  445. EXT4_ERROR_INODE(orig_inode,
  446. "new_ext_end(%u) should be less than or equal to "
  447. "oext->ee_block(%u) + oext_alen(%d) - 1",
  448. new_ext_end, le32_to_cpu(oext->ee_block),
  449. oext_alen);
  450. ret = -EIO;
  451. goto out;
  452. }
  453. /*
  454. * Case: new_ext is smaller than original extent
  455. * oext |---------------|
  456. * new_ext |-----------|
  457. * end_ext |---|
  458. */
  459. if (le32_to_cpu(oext->ee_block) <= new_ext_end &&
  460. new_ext_end < le32_to_cpu(oext->ee_block) + oext_alen - 1) {
  461. end_ext.ee_len =
  462. cpu_to_le16(le32_to_cpu(oext->ee_block) +
  463. oext_alen - 1 - new_ext_end);
  464. copy_extent_status(oext, &end_ext);
  465. end_ext_alen = ext4_ext_get_actual_len(&end_ext);
  466. ext4_ext_store_pblock(&end_ext,
  467. (ext4_ext_pblock(o_end) + oext_alen - end_ext_alen));
  468. end_ext.ee_block =
  469. cpu_to_le32(le32_to_cpu(o_end->ee_block) +
  470. oext_alen - end_ext_alen);
  471. }
  472. ret = mext_insert_extents(handle, orig_inode, orig_path, o_start,
  473. o_end, &start_ext, &new_ext, &end_ext);
  474. out:
  475. return ret;
  476. }
  477. /**
  478. * mext_calc_swap_extents - Calculate extents for extent swapping.
  479. *
  480. * @tmp_dext: the extent that will belong to the original inode
  481. * @tmp_oext: the extent that will belong to the donor inode
  482. * @orig_off: block offset of original inode
  483. * @donor_off: block offset of donor inode
  484. * @max_count: the maximum length of extents
  485. *
  486. * Return 0 on success, or a negative error value on failure.
  487. */
  488. static int
  489. mext_calc_swap_extents(struct ext4_extent *tmp_dext,
  490. struct ext4_extent *tmp_oext,
  491. ext4_lblk_t orig_off, ext4_lblk_t donor_off,
  492. ext4_lblk_t max_count)
  493. {
  494. ext4_lblk_t diff, orig_diff;
  495. struct ext4_extent dext_old, oext_old;
  496. BUG_ON(orig_off != donor_off);
  497. /* original and donor extents have to cover the same block offset */
  498. if (orig_off < le32_to_cpu(tmp_oext->ee_block) ||
  499. le32_to_cpu(tmp_oext->ee_block) +
  500. ext4_ext_get_actual_len(tmp_oext) - 1 < orig_off)
  501. return -ENODATA;
  502. if (orig_off < le32_to_cpu(tmp_dext->ee_block) ||
  503. le32_to_cpu(tmp_dext->ee_block) +
  504. ext4_ext_get_actual_len(tmp_dext) - 1 < orig_off)
  505. return -ENODATA;
  506. dext_old = *tmp_dext;
  507. oext_old = *tmp_oext;
  508. /* When tmp_dext is too large, pick up the target range. */
  509. diff = donor_off - le32_to_cpu(tmp_dext->ee_block);
  510. ext4_ext_store_pblock(tmp_dext, ext4_ext_pblock(tmp_dext) + diff);
  511. tmp_dext->ee_block =
  512. cpu_to_le32(le32_to_cpu(tmp_dext->ee_block) + diff);
  513. tmp_dext->ee_len = cpu_to_le16(le16_to_cpu(tmp_dext->ee_len) - diff);
  514. if (max_count < ext4_ext_get_actual_len(tmp_dext))
  515. tmp_dext->ee_len = cpu_to_le16(max_count);
  516. orig_diff = orig_off - le32_to_cpu(tmp_oext->ee_block);
  517. ext4_ext_store_pblock(tmp_oext, ext4_ext_pblock(tmp_oext) + orig_diff);
  518. /* Adjust extent length if donor extent is larger than orig */
  519. if (ext4_ext_get_actual_len(tmp_dext) >
  520. ext4_ext_get_actual_len(tmp_oext) - orig_diff)
  521. tmp_dext->ee_len = cpu_to_le16(le16_to_cpu(tmp_oext->ee_len) -
  522. orig_diff);
  523. tmp_oext->ee_len = cpu_to_le16(ext4_ext_get_actual_len(tmp_dext));
  524. copy_extent_status(&oext_old, tmp_dext);
  525. copy_extent_status(&dext_old, tmp_oext);
  526. return 0;
  527. }
  528. /**
  529. * mext_check_coverage - Check that all extents in range has the same type
  530. *
  531. * @inode: inode in question
  532. * @from: block offset of inode
  533. * @count: block count to be checked
  534. * @uninit: extents expected to be uninitialized
  535. * @err: pointer to save error value
  536. *
  537. * Return 1 if all extents in range has expected type, and zero otherwise.
  538. */
  539. static int
  540. mext_check_coverage(struct inode *inode, ext4_lblk_t from, ext4_lblk_t count,
  541. int uninit, int *err)
  542. {
  543. struct ext4_ext_path *path = NULL;
  544. struct ext4_extent *ext;
  545. ext4_lblk_t last = from + count;
  546. while (from < last) {
  547. *err = get_ext_path(inode, from, &path);
  548. if (*err)
  549. return 0;
  550. ext = path[ext_depth(inode)].p_ext;
  551. if (!ext) {
  552. ext4_ext_drop_refs(path);
  553. return 0;
  554. }
  555. if (uninit != ext4_ext_is_uninitialized(ext)) {
  556. ext4_ext_drop_refs(path);
  557. return 0;
  558. }
  559. from += ext4_ext_get_actual_len(ext);
  560. ext4_ext_drop_refs(path);
  561. }
  562. return 1;
  563. }
  564. /**
  565. * mext_replace_branches - Replace original extents with new extents
  566. *
  567. * @handle: journal handle
  568. * @orig_inode: original inode
  569. * @donor_inode: donor inode
  570. * @from: block offset of orig_inode
  571. * @count: block count to be replaced
  572. * @err: pointer to save return value
  573. *
  574. * Replace original inode extents and donor inode extents page by page.
  575. * We implement this replacement in the following three steps:
  576. * 1. Save the block information of original and donor inodes into
  577. * dummy extents.
  578. * 2. Change the block information of original inode to point at the
  579. * donor inode blocks.
  580. * 3. Change the block information of donor inode to point at the saved
  581. * original inode blocks in the dummy extents.
  582. *
  583. * Return replaced block count.
  584. */
  585. static int
  586. mext_replace_branches(handle_t *handle, struct inode *orig_inode,
  587. struct inode *donor_inode, ext4_lblk_t from,
  588. ext4_lblk_t count, int *err)
  589. {
  590. struct ext4_ext_path *orig_path = NULL;
  591. struct ext4_ext_path *donor_path = NULL;
  592. struct ext4_extent *oext, *dext;
  593. struct ext4_extent tmp_dext, tmp_oext;
  594. ext4_lblk_t orig_off = from, donor_off = from;
  595. int depth;
  596. int replaced_count = 0;
  597. int dext_alen;
  598. /* Get the original extent for the block "orig_off" */
  599. *err = get_ext_path(orig_inode, orig_off, &orig_path);
  600. if (*err)
  601. goto out;
  602. /* Get the donor extent for the head */
  603. *err = get_ext_path(donor_inode, donor_off, &donor_path);
  604. if (*err)
  605. goto out;
  606. depth = ext_depth(orig_inode);
  607. oext = orig_path[depth].p_ext;
  608. tmp_oext = *oext;
  609. depth = ext_depth(donor_inode);
  610. dext = donor_path[depth].p_ext;
  611. tmp_dext = *dext;
  612. *err = mext_calc_swap_extents(&tmp_dext, &tmp_oext, orig_off,
  613. donor_off, count);
  614. if (*err)
  615. goto out;
  616. /* Loop for the donor extents */
  617. while (1) {
  618. /* The extent for donor must be found. */
  619. if (!dext) {
  620. EXT4_ERROR_INODE(donor_inode,
  621. "The extent for donor must be found");
  622. *err = -EIO;
  623. goto out;
  624. } else if (donor_off != le32_to_cpu(tmp_dext.ee_block)) {
  625. EXT4_ERROR_INODE(donor_inode,
  626. "Donor offset(%u) and the first block of donor "
  627. "extent(%u) should be equal",
  628. donor_off,
  629. le32_to_cpu(tmp_dext.ee_block));
  630. *err = -EIO;
  631. goto out;
  632. }
  633. /* Set donor extent to orig extent */
  634. *err = mext_leaf_block(handle, orig_inode,
  635. orig_path, &tmp_dext, &orig_off);
  636. if (*err)
  637. goto out;
  638. /* Set orig extent to donor extent */
  639. *err = mext_leaf_block(handle, donor_inode,
  640. donor_path, &tmp_oext, &donor_off);
  641. if (*err)
  642. goto out;
  643. dext_alen = ext4_ext_get_actual_len(&tmp_dext);
  644. replaced_count += dext_alen;
  645. donor_off += dext_alen;
  646. orig_off += dext_alen;
  647. /* Already moved the expected blocks */
  648. if (replaced_count >= count)
  649. break;
  650. if (orig_path)
  651. ext4_ext_drop_refs(orig_path);
  652. *err = get_ext_path(orig_inode, orig_off, &orig_path);
  653. if (*err)
  654. goto out;
  655. depth = ext_depth(orig_inode);
  656. oext = orig_path[depth].p_ext;
  657. tmp_oext = *oext;
  658. if (donor_path)
  659. ext4_ext_drop_refs(donor_path);
  660. *err = get_ext_path(donor_inode, donor_off, &donor_path);
  661. if (*err)
  662. goto out;
  663. depth = ext_depth(donor_inode);
  664. dext = donor_path[depth].p_ext;
  665. tmp_dext = *dext;
  666. *err = mext_calc_swap_extents(&tmp_dext, &tmp_oext, orig_off,
  667. donor_off, count - replaced_count);
  668. if (*err)
  669. goto out;
  670. }
  671. out:
  672. if (orig_path) {
  673. ext4_ext_drop_refs(orig_path);
  674. kfree(orig_path);
  675. }
  676. if (donor_path) {
  677. ext4_ext_drop_refs(donor_path);
  678. kfree(donor_path);
  679. }
  680. ext4_ext_invalidate_cache(orig_inode);
  681. ext4_ext_invalidate_cache(donor_inode);
  682. return replaced_count;
  683. }
  684. /**
  685. * mext_page_double_lock - Grab and lock pages on both @inode1 and @inode2
  686. *
  687. * @inode1: the inode structure
  688. * @inode2: the inode structure
  689. * @index: page index
  690. * @page: result page vector
  691. *
  692. * Grab two locked pages for inode's by inode order
  693. */
  694. static int
  695. mext_page_double_lock(struct inode *inode1, struct inode *inode2,
  696. pgoff_t index, struct page *page[2])
  697. {
  698. struct address_space *mapping[2];
  699. unsigned fl = AOP_FLAG_NOFS;
  700. BUG_ON(!inode1 || !inode2);
  701. if (inode1 < inode2) {
  702. mapping[0] = inode1->i_mapping;
  703. mapping[1] = inode2->i_mapping;
  704. } else {
  705. mapping[0] = inode2->i_mapping;
  706. mapping[1] = inode1->i_mapping;
  707. }
  708. page[0] = grab_cache_page_write_begin(mapping[0], index, fl);
  709. if (!page[0])
  710. return -ENOMEM;
  711. page[1] = grab_cache_page_write_begin(mapping[1], index, fl);
  712. if (!page[1]) {
  713. unlock_page(page[0]);
  714. page_cache_release(page[0]);
  715. return -ENOMEM;
  716. }
  717. if (inode1 > inode2) {
  718. struct page *tmp;
  719. tmp = page[0];
  720. page[0] = page[1];
  721. page[1] = tmp;
  722. }
  723. return 0;
  724. }
  725. /* Force page buffers uptodate w/o dropping page's lock */
  726. static int
  727. mext_page_mkuptodate(struct page *page, unsigned from, unsigned to)
  728. {
  729. struct inode *inode = page->mapping->host;
  730. sector_t block;
  731. struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
  732. unsigned int blocksize, block_start, block_end;
  733. int i, err, nr = 0, partial = 0;
  734. BUG_ON(!PageLocked(page));
  735. BUG_ON(PageWriteback(page));
  736. if (PageUptodate(page))
  737. return 0;
  738. blocksize = 1 << inode->i_blkbits;
  739. if (!page_has_buffers(page))
  740. create_empty_buffers(page, blocksize, 0);
  741. head = page_buffers(page);
  742. block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
  743. for (bh = head, block_start = 0; bh != head || !block_start;
  744. block++, block_start = block_end, bh = bh->b_this_page) {
  745. block_end = block_start + blocksize;
  746. if (block_end <= from || block_start >= to) {
  747. if (!buffer_uptodate(bh))
  748. partial = 1;
  749. continue;
  750. }
  751. if (buffer_uptodate(bh))
  752. continue;
  753. if (!buffer_mapped(bh)) {
  754. int err = 0;
  755. err = ext4_get_block(inode, block, bh, 0);
  756. if (err) {
  757. SetPageError(page);
  758. return err;
  759. }
  760. if (!buffer_mapped(bh)) {
  761. zero_user(page, block_start, blocksize);
  762. if (!err)
  763. set_buffer_uptodate(bh);
  764. continue;
  765. }
  766. }
  767. BUG_ON(nr >= MAX_BUF_PER_PAGE);
  768. arr[nr++] = bh;
  769. }
  770. /* No io required */
  771. if (!nr)
  772. goto out;
  773. for (i = 0; i < nr; i++) {
  774. bh = arr[i];
  775. if (!bh_uptodate_or_lock(bh)) {
  776. err = bh_submit_read(bh);
  777. if (err)
  778. return err;
  779. }
  780. }
  781. out:
  782. if (!partial)
  783. SetPageUptodate(page);
  784. return 0;
  785. }
  786. /**
  787. * move_extent_per_page - Move extent data per page
  788. *
  789. * @o_filp: file structure of original file
  790. * @donor_inode: donor inode
  791. * @orig_page_offset: page index on original file
  792. * @data_offset_in_page: block index where data swapping starts
  793. * @block_len_in_page: the number of blocks to be swapped
  794. * @uninit: orig extent is uninitialized or not
  795. * @err: pointer to save return value
  796. *
  797. * Save the data in original inode blocks and replace original inode extents
  798. * with donor inode extents by calling mext_replace_branches().
  799. * Finally, write out the saved data in new original inode blocks. Return
  800. * replaced block count.
  801. */
  802. static int
  803. move_extent_per_page(struct file *o_filp, struct inode *donor_inode,
  804. pgoff_t orig_page_offset, int data_offset_in_page,
  805. int block_len_in_page, int uninit, int *err)
  806. {
  807. struct inode *orig_inode = o_filp->f_dentry->d_inode;
  808. struct page *pagep[2] = {NULL, NULL};
  809. handle_t *handle;
  810. ext4_lblk_t orig_blk_offset;
  811. long long offs = orig_page_offset << PAGE_CACHE_SHIFT;
  812. unsigned long blocksize = orig_inode->i_sb->s_blocksize;
  813. unsigned int w_flags = 0;
  814. unsigned int tmp_data_size, data_size, replaced_size;
  815. int err2, jblocks, retries = 0;
  816. int replaced_count = 0;
  817. int from = data_offset_in_page << orig_inode->i_blkbits;
  818. int blocks_per_page = PAGE_CACHE_SIZE >> orig_inode->i_blkbits;
  819. /*
  820. * It needs twice the amount of ordinary journal buffers because
  821. * inode and donor_inode may change each different metadata blocks.
  822. */
  823. again:
  824. *err = 0;
  825. jblocks = ext4_writepage_trans_blocks(orig_inode) * 2;
  826. handle = ext4_journal_start(orig_inode, jblocks);
  827. if (IS_ERR(handle)) {
  828. *err = PTR_ERR(handle);
  829. return 0;
  830. }
  831. if (segment_eq(get_fs(), KERNEL_DS))
  832. w_flags |= AOP_FLAG_UNINTERRUPTIBLE;
  833. orig_blk_offset = orig_page_offset * blocks_per_page +
  834. data_offset_in_page;
  835. offs = (long long)orig_blk_offset << orig_inode->i_blkbits;
  836. /* Calculate data_size */
  837. if ((orig_blk_offset + block_len_in_page - 1) ==
  838. ((orig_inode->i_size - 1) >> orig_inode->i_blkbits)) {
  839. /* Replace the last block */
  840. tmp_data_size = orig_inode->i_size & (blocksize - 1);
  841. /*
  842. * If data_size equal zero, it shows data_size is multiples of
  843. * blocksize. So we set appropriate value.
  844. */
  845. if (tmp_data_size == 0)
  846. tmp_data_size = blocksize;
  847. data_size = tmp_data_size +
  848. ((block_len_in_page - 1) << orig_inode->i_blkbits);
  849. } else
  850. data_size = block_len_in_page << orig_inode->i_blkbits;
  851. replaced_size = data_size;
  852. *err = mext_page_double_lock(orig_inode, donor_inode, orig_page_offset,
  853. pagep);
  854. if (unlikely(*err < 0))
  855. goto stop_journal;
  856. /*
  857. * If orig extent was uninitialized it can become initialized
  858. * at any time after i_data_sem was dropped, in order to
  859. * serialize with delalloc we have recheck extent while we
  860. * hold page's lock, if it is still the case data copy is not
  861. * necessary, just swap data blocks between orig and donor.
  862. */
  863. if (uninit) {
  864. double_down_write_data_sem(orig_inode, donor_inode);
  865. /* If any of extents in range became initialized we have to
  866. * fallback to data copying */
  867. uninit = mext_check_coverage(orig_inode, orig_blk_offset,
  868. block_len_in_page, 1, err);
  869. if (*err)
  870. goto drop_data_sem;
  871. uninit &= mext_check_coverage(donor_inode, orig_blk_offset,
  872. block_len_in_page, 1, err);
  873. if (*err)
  874. goto drop_data_sem;
  875. if (!uninit) {
  876. double_up_write_data_sem(orig_inode, donor_inode);
  877. goto data_copy;
  878. }
  879. if ((page_has_private(pagep[0]) &&
  880. !try_to_release_page(pagep[0], 0)) ||
  881. (page_has_private(pagep[1]) &&
  882. !try_to_release_page(pagep[1], 0))) {
  883. *err = -EBUSY;
  884. goto drop_data_sem;
  885. }
  886. replaced_count = mext_replace_branches(handle, orig_inode,
  887. donor_inode, orig_blk_offset,
  888. block_len_in_page, err);
  889. drop_data_sem:
  890. double_up_write_data_sem(orig_inode, donor_inode);
  891. goto unlock_pages;
  892. }
  893. data_copy:
  894. *err = mext_page_mkuptodate(pagep[0], from, from + replaced_size);
  895. if (*err)
  896. goto unlock_pages;
  897. /* At this point all buffers in range are uptodate, old mapping layout
  898. * is no longer required, try to drop it now. */
  899. if ((page_has_private(pagep[0]) && !try_to_release_page(pagep[0], 0)) ||
  900. (page_has_private(pagep[1]) && !try_to_release_page(pagep[1], 0))) {
  901. *err = -EBUSY;
  902. goto unlock_pages;
  903. }
  904. replaced_count = mext_replace_branches(handle, orig_inode, donor_inode,
  905. orig_blk_offset,
  906. block_len_in_page, err);
  907. if (*err) {
  908. if (replaced_count) {
  909. block_len_in_page = replaced_count;
  910. replaced_size =
  911. block_len_in_page << orig_inode->i_blkbits;
  912. } else
  913. goto unlock_pages;
  914. }
  915. /* Perform all necessary steps similar write_begin()/write_end()
  916. * but keeping in mind that i_size will not change */
  917. *err = __block_write_begin(pagep[0], from, from + replaced_size,
  918. ext4_get_block);
  919. if (!*err)
  920. *err = block_commit_write(pagep[0], from, from + replaced_size);
  921. if (unlikely(*err < 0))
  922. goto repair_branches;
  923. /* Even in case of data=writeback it is reasonable to pin
  924. * inode to transaction, to prevent unexpected data loss */
  925. *err = ext4_jbd2_file_inode(handle, orig_inode);
  926. unlock_pages:
  927. unlock_page(pagep[0]);
  928. page_cache_release(pagep[0]);
  929. unlock_page(pagep[1]);
  930. page_cache_release(pagep[1]);
  931. stop_journal:
  932. ext4_journal_stop(handle);
  933. /* Buffer was busy because probably is pinned to journal transaction,
  934. * force transaction commit may help to free it. */
  935. if (*err == -EBUSY && ext4_should_retry_alloc(orig_inode->i_sb,
  936. &retries))
  937. goto again;
  938. return replaced_count;
  939. repair_branches:
  940. /*
  941. * This should never ever happen!
  942. * Extents are swapped already, but we are not able to copy data.
  943. * Try to swap extents to it's original places
  944. */
  945. double_down_write_data_sem(orig_inode, donor_inode);
  946. replaced_count = mext_replace_branches(handle, donor_inode, orig_inode,
  947. orig_blk_offset,
  948. block_len_in_page, &err2);
  949. double_up_write_data_sem(orig_inode, donor_inode);
  950. if (replaced_count != block_len_in_page) {
  951. EXT4_ERROR_INODE_BLOCK(orig_inode, (sector_t)(orig_blk_offset),
  952. "Unable to copy data block,"
  953. " data will be lost.");
  954. *err = -EIO;
  955. }
  956. replaced_count = 0;
  957. goto unlock_pages;
  958. }
  959. /**
  960. * mext_check_arguments - Check whether move extent can be done
  961. *
  962. * @orig_inode: original inode
  963. * @donor_inode: donor inode
  964. * @orig_start: logical start offset in block for orig
  965. * @donor_start: logical start offset in block for donor
  966. * @len: the number of blocks to be moved
  967. *
  968. * Check the arguments of ext4_move_extents() whether the files can be
  969. * exchanged with each other.
  970. * Return 0 on success, or a negative error value on failure.
  971. */
  972. static int
  973. mext_check_arguments(struct inode *orig_inode,
  974. struct inode *donor_inode, __u64 orig_start,
  975. __u64 donor_start, __u64 *len)
  976. {
  977. ext4_lblk_t orig_blocks, donor_blocks;
  978. unsigned int blkbits = orig_inode->i_blkbits;
  979. unsigned int blocksize = 1 << blkbits;
  980. if (donor_inode->i_mode & (S_ISUID|S_ISGID)) {
  981. ext4_debug("ext4 move extent: suid or sgid is set"
  982. " to donor file [ino:orig %lu, donor %lu]\n",
  983. orig_inode->i_ino, donor_inode->i_ino);
  984. return -EINVAL;
  985. }
  986. if (IS_IMMUTABLE(donor_inode) || IS_APPEND(donor_inode))
  987. return -EPERM;
  988. /* Ext4 move extent does not support swapfile */
  989. if (IS_SWAPFILE(orig_inode) || IS_SWAPFILE(donor_inode)) {
  990. ext4_debug("ext4 move extent: The argument files should "
  991. "not be swapfile [ino:orig %lu, donor %lu]\n",
  992. orig_inode->i_ino, donor_inode->i_ino);
  993. return -EINVAL;
  994. }
  995. /* Ext4 move extent supports only extent based file */
  996. if (!(ext4_test_inode_flag(orig_inode, EXT4_INODE_EXTENTS))) {
  997. ext4_debug("ext4 move extent: orig file is not extents "
  998. "based file [ino:orig %lu]\n", orig_inode->i_ino);
  999. return -EOPNOTSUPP;
  1000. } else if (!(ext4_test_inode_flag(donor_inode, EXT4_INODE_EXTENTS))) {
  1001. ext4_debug("ext4 move extent: donor file is not extents "
  1002. "based file [ino:donor %lu]\n", donor_inode->i_ino);
  1003. return -EOPNOTSUPP;
  1004. }
  1005. if ((!orig_inode->i_size) || (!donor_inode->i_size)) {
  1006. ext4_debug("ext4 move extent: File size is 0 byte\n");
  1007. return -EINVAL;
  1008. }
  1009. /* Start offset should be same */
  1010. if (orig_start != donor_start) {
  1011. ext4_debug("ext4 move extent: orig and donor's start "
  1012. "offset are not same [ino:orig %lu, donor %lu]\n",
  1013. orig_inode->i_ino, donor_inode->i_ino);
  1014. return -EINVAL;
  1015. }
  1016. if ((orig_start >= EXT_MAX_BLOCKS) ||
  1017. (*len > EXT_MAX_BLOCKS) ||
  1018. (orig_start + *len >= EXT_MAX_BLOCKS)) {
  1019. ext4_debug("ext4 move extent: Can't handle over [%u] blocks "
  1020. "[ino:orig %lu, donor %lu]\n", EXT_MAX_BLOCKS,
  1021. orig_inode->i_ino, donor_inode->i_ino);
  1022. return -EINVAL;
  1023. }
  1024. if (orig_inode->i_size > donor_inode->i_size) {
  1025. donor_blocks = (donor_inode->i_size + blocksize - 1) >> blkbits;
  1026. /* TODO: eliminate this artificial restriction */
  1027. if (orig_start >= donor_blocks) {
  1028. ext4_debug("ext4 move extent: orig start offset "
  1029. "[%llu] should be less than donor file blocks "
  1030. "[%u] [ino:orig %lu, donor %lu]\n",
  1031. orig_start, donor_blocks,
  1032. orig_inode->i_ino, donor_inode->i_ino);
  1033. return -EINVAL;
  1034. }
  1035. /* TODO: eliminate this artificial restriction */
  1036. if (orig_start + *len > donor_blocks) {
  1037. ext4_debug("ext4 move extent: End offset [%llu] should "
  1038. "be less than donor file blocks [%u]."
  1039. "So adjust length from %llu to %llu "
  1040. "[ino:orig %lu, donor %lu]\n",
  1041. orig_start + *len, donor_blocks,
  1042. *len, donor_blocks - orig_start,
  1043. orig_inode->i_ino, donor_inode->i_ino);
  1044. *len = donor_blocks - orig_start;
  1045. }
  1046. } else {
  1047. orig_blocks = (orig_inode->i_size + blocksize - 1) >> blkbits;
  1048. if (orig_start >= orig_blocks) {
  1049. ext4_debug("ext4 move extent: start offset [%llu] "
  1050. "should be less than original file blocks "
  1051. "[%u] [ino:orig %lu, donor %lu]\n",
  1052. orig_start, orig_blocks,
  1053. orig_inode->i_ino, donor_inode->i_ino);
  1054. return -EINVAL;
  1055. }
  1056. if (orig_start + *len > orig_blocks) {
  1057. ext4_debug("ext4 move extent: Adjust length "
  1058. "from %llu to %llu. Because it should be "
  1059. "less than original file blocks "
  1060. "[ino:orig %lu, donor %lu]\n",
  1061. *len, orig_blocks - orig_start,
  1062. orig_inode->i_ino, donor_inode->i_ino);
  1063. *len = orig_blocks - orig_start;
  1064. }
  1065. }
  1066. if (!*len) {
  1067. ext4_debug("ext4 move extent: len should not be 0 "
  1068. "[ino:orig %lu, donor %lu]\n", orig_inode->i_ino,
  1069. donor_inode->i_ino);
  1070. return -EINVAL;
  1071. }
  1072. return 0;
  1073. }
  1074. /**
  1075. * mext_inode_double_lock - Lock i_mutex on both @inode1 and @inode2
  1076. *
  1077. * @inode1: the inode structure
  1078. * @inode2: the inode structure
  1079. *
  1080. * Lock two inodes' i_mutex
  1081. */
  1082. static void
  1083. mext_inode_double_lock(struct inode *inode1, struct inode *inode2)
  1084. {
  1085. BUG_ON(inode1 == inode2);
  1086. if (inode1 < inode2) {
  1087. mutex_lock_nested(&inode1->i_mutex, I_MUTEX_PARENT);
  1088. mutex_lock_nested(&inode2->i_mutex, I_MUTEX_CHILD);
  1089. } else {
  1090. mutex_lock_nested(&inode2->i_mutex, I_MUTEX_PARENT);
  1091. mutex_lock_nested(&inode1->i_mutex, I_MUTEX_CHILD);
  1092. }
  1093. }
  1094. /**
  1095. * mext_inode_double_unlock - Release i_mutex on both @inode1 and @inode2
  1096. *
  1097. * @inode1: the inode that is released first
  1098. * @inode2: the inode that is released second
  1099. *
  1100. */
  1101. static void
  1102. mext_inode_double_unlock(struct inode *inode1, struct inode *inode2)
  1103. {
  1104. mutex_unlock(&inode1->i_mutex);
  1105. mutex_unlock(&inode2->i_mutex);
  1106. }
  1107. /**
  1108. * ext4_move_extents - Exchange the specified range of a file
  1109. *
  1110. * @o_filp: file structure of the original file
  1111. * @d_filp: file structure of the donor file
  1112. * @orig_start: start offset in block for orig
  1113. * @donor_start: start offset in block for donor
  1114. * @len: the number of blocks to be moved
  1115. * @moved_len: moved block length
  1116. *
  1117. * This function returns 0 and moved block length is set in moved_len
  1118. * if succeed, otherwise returns error value.
  1119. *
  1120. * Note: ext4_move_extents() proceeds the following order.
  1121. * 1:ext4_move_extents() calculates the last block number of moving extent
  1122. * function by the start block number (orig_start) and the number of blocks
  1123. * to be moved (len) specified as arguments.
  1124. * If the {orig, donor}_start points a hole, the extent's start offset
  1125. * pointed by ext_cur (current extent), holecheck_path, orig_path are set
  1126. * after hole behind.
  1127. * 2:Continue step 3 to step 5, until the holecheck_path points to last_extent
  1128. * or the ext_cur exceeds the block_end which is last logical block number.
  1129. * 3:To get the length of continues area, call mext_next_extent()
  1130. * specified with the ext_cur (initial value is holecheck_path) re-cursive,
  1131. * until find un-continuous extent, the start logical block number exceeds
  1132. * the block_end or the extent points to the last extent.
  1133. * 4:Exchange the original inode data with donor inode data
  1134. * from orig_page_offset to seq_end_page.
  1135. * The start indexes of data are specified as arguments.
  1136. * That of the original inode is orig_page_offset,
  1137. * and the donor inode is also orig_page_offset
  1138. * (To easily handle blocksize != pagesize case, the offset for the
  1139. * donor inode is block unit).
  1140. * 5:Update holecheck_path and orig_path to points a next proceeding extent,
  1141. * then returns to step 2.
  1142. * 6:Release holecheck_path, orig_path and set the len to moved_len
  1143. * which shows the number of moved blocks.
  1144. * The moved_len is useful for the command to calculate the file offset
  1145. * for starting next move extent ioctl.
  1146. * 7:Return 0 on success, or a negative error value on failure.
  1147. */
  1148. int
  1149. ext4_move_extents(struct file *o_filp, struct file *d_filp,
  1150. __u64 orig_start, __u64 donor_start, __u64 len,
  1151. __u64 *moved_len)
  1152. {
  1153. struct inode *orig_inode = o_filp->f_dentry->d_inode;
  1154. struct inode *donor_inode = d_filp->f_dentry->d_inode;
  1155. struct ext4_ext_path *orig_path = NULL, *holecheck_path = NULL;
  1156. struct ext4_extent *ext_prev, *ext_cur, *ext_dummy;
  1157. ext4_lblk_t block_start = orig_start;
  1158. ext4_lblk_t block_end, seq_start, add_blocks, file_end, seq_blocks = 0;
  1159. ext4_lblk_t rest_blocks;
  1160. pgoff_t orig_page_offset = 0, seq_end_page;
  1161. int ret, depth, last_extent = 0;
  1162. int blocks_per_page = PAGE_CACHE_SIZE >> orig_inode->i_blkbits;
  1163. int data_offset_in_page;
  1164. int block_len_in_page;
  1165. int uninit;
  1166. if (orig_inode->i_sb != donor_inode->i_sb) {
  1167. ext4_debug("ext4 move extent: The argument files "
  1168. "should be in same FS [ino:orig %lu, donor %lu]\n",
  1169. orig_inode->i_ino, donor_inode->i_ino);
  1170. return -EINVAL;
  1171. }
  1172. /* orig and donor should be different inodes */
  1173. if (orig_inode == donor_inode) {
  1174. ext4_debug("ext4 move extent: The argument files should not "
  1175. "be same inode [ino:orig %lu, donor %lu]\n",
  1176. orig_inode->i_ino, donor_inode->i_ino);
  1177. return -EINVAL;
  1178. }
  1179. /* Regular file check */
  1180. if (!S_ISREG(orig_inode->i_mode) || !S_ISREG(donor_inode->i_mode)) {
  1181. ext4_debug("ext4 move extent: The argument files should be "
  1182. "regular file [ino:orig %lu, donor %lu]\n",
  1183. orig_inode->i_ino, donor_inode->i_ino);
  1184. return -EINVAL;
  1185. }
  1186. /* TODO: This is non obvious task to swap blocks for inodes with full
  1187. jornaling enabled */
  1188. if (ext4_should_journal_data(orig_inode) ||
  1189. ext4_should_journal_data(donor_inode)) {
  1190. return -EINVAL;
  1191. }
  1192. /* Protect orig and donor inodes against a truncate */
  1193. mext_inode_double_lock(orig_inode, donor_inode);
  1194. /* Protect extent tree against block allocations via delalloc */
  1195. double_down_write_data_sem(orig_inode, donor_inode);
  1196. /* Check the filesystem environment whether move_extent can be done */
  1197. ret = mext_check_arguments(orig_inode, donor_inode, orig_start,
  1198. donor_start, &len);
  1199. if (ret)
  1200. goto out;
  1201. file_end = (i_size_read(orig_inode) - 1) >> orig_inode->i_blkbits;
  1202. block_end = block_start + len - 1;
  1203. if (file_end < block_end)
  1204. len -= block_end - file_end;
  1205. ret = get_ext_path(orig_inode, block_start, &orig_path);
  1206. if (ret)
  1207. goto out;
  1208. /* Get path structure to check the hole */
  1209. ret = get_ext_path(orig_inode, block_start, &holecheck_path);
  1210. if (ret)
  1211. goto out;
  1212. depth = ext_depth(orig_inode);
  1213. ext_cur = holecheck_path[depth].p_ext;
  1214. /*
  1215. * Get proper starting location of block replacement if block_start was
  1216. * within the hole.
  1217. */
  1218. if (le32_to_cpu(ext_cur->ee_block) +
  1219. ext4_ext_get_actual_len(ext_cur) - 1 < block_start) {
  1220. /*
  1221. * The hole exists between extents or the tail of
  1222. * original file.
  1223. */
  1224. last_extent = mext_next_extent(orig_inode,
  1225. holecheck_path, &ext_cur);
  1226. if (last_extent < 0) {
  1227. ret = last_extent;
  1228. goto out;
  1229. }
  1230. last_extent = mext_next_extent(orig_inode, orig_path,
  1231. &ext_dummy);
  1232. if (last_extent < 0) {
  1233. ret = last_extent;
  1234. goto out;
  1235. }
  1236. seq_start = le32_to_cpu(ext_cur->ee_block);
  1237. } else if (le32_to_cpu(ext_cur->ee_block) > block_start)
  1238. /* The hole exists at the beginning of original file. */
  1239. seq_start = le32_to_cpu(ext_cur->ee_block);
  1240. else
  1241. seq_start = block_start;
  1242. /* No blocks within the specified range. */
  1243. if (le32_to_cpu(ext_cur->ee_block) > block_end) {
  1244. ext4_debug("ext4 move extent: The specified range of file "
  1245. "may be the hole\n");
  1246. ret = -EINVAL;
  1247. goto out;
  1248. }
  1249. /* Adjust start blocks */
  1250. add_blocks = min(le32_to_cpu(ext_cur->ee_block) +
  1251. ext4_ext_get_actual_len(ext_cur), block_end + 1) -
  1252. max(le32_to_cpu(ext_cur->ee_block), block_start);
  1253. while (!last_extent && le32_to_cpu(ext_cur->ee_block) <= block_end) {
  1254. seq_blocks += add_blocks;
  1255. /* Adjust tail blocks */
  1256. if (seq_start + seq_blocks - 1 > block_end)
  1257. seq_blocks = block_end - seq_start + 1;
  1258. ext_prev = ext_cur;
  1259. last_extent = mext_next_extent(orig_inode, holecheck_path,
  1260. &ext_cur);
  1261. if (last_extent < 0) {
  1262. ret = last_extent;
  1263. break;
  1264. }
  1265. add_blocks = ext4_ext_get_actual_len(ext_cur);
  1266. /*
  1267. * Extend the length of contiguous block (seq_blocks)
  1268. * if extents are contiguous.
  1269. */
  1270. if (ext4_can_extents_be_merged(orig_inode,
  1271. ext_prev, ext_cur) &&
  1272. block_end >= le32_to_cpu(ext_cur->ee_block) &&
  1273. !last_extent)
  1274. continue;
  1275. /* Is original extent is uninitialized */
  1276. uninit = ext4_ext_is_uninitialized(ext_prev);
  1277. data_offset_in_page = seq_start % blocks_per_page;
  1278. /*
  1279. * Calculate data blocks count that should be swapped
  1280. * at the first page.
  1281. */
  1282. if (data_offset_in_page + seq_blocks > blocks_per_page) {
  1283. /* Swapped blocks are across pages */
  1284. block_len_in_page =
  1285. blocks_per_page - data_offset_in_page;
  1286. } else {
  1287. /* Swapped blocks are in a page */
  1288. block_len_in_page = seq_blocks;
  1289. }
  1290. orig_page_offset = seq_start >>
  1291. (PAGE_CACHE_SHIFT - orig_inode->i_blkbits);
  1292. seq_end_page = (seq_start + seq_blocks - 1) >>
  1293. (PAGE_CACHE_SHIFT - orig_inode->i_blkbits);
  1294. seq_start = le32_to_cpu(ext_cur->ee_block);
  1295. rest_blocks = seq_blocks;
  1296. /*
  1297. * Up semaphore to avoid following problems:
  1298. * a. transaction deadlock among ext4_journal_start,
  1299. * ->write_begin via pagefault, and jbd2_journal_commit
  1300. * b. racing with ->readpage, ->write_begin, and ext4_get_block
  1301. * in move_extent_per_page
  1302. */
  1303. double_up_write_data_sem(orig_inode, donor_inode);
  1304. while (orig_page_offset <= seq_end_page) {
  1305. /* Swap original branches with new branches */
  1306. block_len_in_page = move_extent_per_page(
  1307. o_filp, donor_inode,
  1308. orig_page_offset,
  1309. data_offset_in_page,
  1310. block_len_in_page, uninit,
  1311. &ret);
  1312. /* Count how many blocks we have exchanged */
  1313. *moved_len += block_len_in_page;
  1314. if (ret < 0)
  1315. break;
  1316. if (*moved_len > len) {
  1317. EXT4_ERROR_INODE(orig_inode,
  1318. "We replaced blocks too much! "
  1319. "sum of replaced: %llu requested: %llu",
  1320. *moved_len, len);
  1321. ret = -EIO;
  1322. break;
  1323. }
  1324. orig_page_offset++;
  1325. data_offset_in_page = 0;
  1326. rest_blocks -= block_len_in_page;
  1327. if (rest_blocks > blocks_per_page)
  1328. block_len_in_page = blocks_per_page;
  1329. else
  1330. block_len_in_page = rest_blocks;
  1331. }
  1332. double_down_write_data_sem(orig_inode, donor_inode);
  1333. if (ret < 0)
  1334. break;
  1335. /* Decrease buffer counter */
  1336. if (holecheck_path)
  1337. ext4_ext_drop_refs(holecheck_path);
  1338. ret = get_ext_path(orig_inode, seq_start, &holecheck_path);
  1339. if (ret)
  1340. break;
  1341. depth = holecheck_path->p_depth;
  1342. /* Decrease buffer counter */
  1343. if (orig_path)
  1344. ext4_ext_drop_refs(orig_path);
  1345. ret = get_ext_path(orig_inode, seq_start, &orig_path);
  1346. if (ret)
  1347. break;
  1348. ext_cur = holecheck_path[depth].p_ext;
  1349. add_blocks = ext4_ext_get_actual_len(ext_cur);
  1350. seq_blocks = 0;
  1351. }
  1352. out:
  1353. if (*moved_len) {
  1354. ext4_discard_preallocations(orig_inode);
  1355. ext4_discard_preallocations(donor_inode);
  1356. }
  1357. if (orig_path) {
  1358. ext4_ext_drop_refs(orig_path);
  1359. kfree(orig_path);
  1360. }
  1361. if (holecheck_path) {
  1362. ext4_ext_drop_refs(holecheck_path);
  1363. kfree(holecheck_path);
  1364. }
  1365. double_up_write_data_sem(orig_inode, donor_inode);
  1366. mext_inode_double_unlock(orig_inode, donor_inode);
  1367. return ret;
  1368. }