i915_gem_tiling.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627
  1. /*
  2. * Copyright © 2008 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. * Authors:
  24. * Eric Anholt <eric@anholt.net>
  25. *
  26. */
  27. #include <linux/acpi.h>
  28. #include <linux/pnp.h>
  29. #include "linux/string.h"
  30. #include "linux/bitops.h"
  31. #include "drmP.h"
  32. #include "drm.h"
  33. #include "i915_drm.h"
  34. #include "i915_drv.h"
  35. /** @file i915_gem_tiling.c
  36. *
  37. * Support for managing tiling state of buffer objects.
  38. *
  39. * The idea behind tiling is to increase cache hit rates by rearranging
  40. * pixel data so that a group of pixel accesses are in the same cacheline.
  41. * Performance improvement from doing this on the back/depth buffer are on
  42. * the order of 30%.
  43. *
  44. * Intel architectures make this somewhat more complicated, though, by
  45. * adjustments made to addressing of data when the memory is in interleaved
  46. * mode (matched pairs of DIMMS) to improve memory bandwidth.
  47. * For interleaved memory, the CPU sends every sequential 64 bytes
  48. * to an alternate memory channel so it can get the bandwidth from both.
  49. *
  50. * The GPU also rearranges its accesses for increased bandwidth to interleaved
  51. * memory, and it matches what the CPU does for non-tiled. However, when tiled
  52. * it does it a little differently, since one walks addresses not just in the
  53. * X direction but also Y. So, along with alternating channels when bit
  54. * 6 of the address flips, it also alternates when other bits flip -- Bits 9
  55. * (every 512 bytes, an X tile scanline) and 10 (every two X tile scanlines)
  56. * are common to both the 915 and 965-class hardware.
  57. *
  58. * The CPU also sometimes XORs in higher bits as well, to improve
  59. * bandwidth doing strided access like we do so frequently in graphics. This
  60. * is called "Channel XOR Randomization" in the MCH documentation. The result
  61. * is that the CPU is XORing in either bit 11 or bit 17 to bit 6 of its address
  62. * decode.
  63. *
  64. * All of this bit 6 XORing has an effect on our memory management,
  65. * as we need to make sure that the 3d driver can correctly address object
  66. * contents.
  67. *
  68. * If we don't have interleaved memory, all tiling is safe and no swizzling is
  69. * required.
  70. *
  71. * When bit 17 is XORed in, we simply refuse to tile at all. Bit
  72. * 17 is not just a page offset, so as we page an objet out and back in,
  73. * individual pages in it will have different bit 17 addresses, resulting in
  74. * each 64 bytes being swapped with its neighbor!
  75. *
  76. * Otherwise, if interleaved, we have to tell the 3d driver what the address
  77. * swizzling it needs to do is, since it's writing with the CPU to the pages
  78. * (bit 6 and potentially bit 11 XORed in), and the GPU is reading from the
  79. * pages (bit 6, 9, and 10 XORed in), resulting in a cumulative bit swizzling
  80. * required by the CPU of XORing in bit 6, 9, 10, and potentially 11, in order
  81. * to match what the GPU expects.
  82. */
  83. #define MCHBAR_I915 0x44
  84. #define MCHBAR_I965 0x48
  85. #define MCHBAR_SIZE (4*4096)
  86. #define DEVEN_REG 0x54
  87. #define DEVEN_MCHBAR_EN (1 << 28)
  88. /* Allocate space for the MCH regs if needed, return nonzero on error */
  89. static int
  90. intel_alloc_mchbar_resource(struct drm_device *dev)
  91. {
  92. struct pci_dev *bridge_dev;
  93. drm_i915_private_t *dev_priv = dev->dev_private;
  94. int reg = IS_I965G(dev) ? MCHBAR_I965 : MCHBAR_I915;
  95. u32 temp_lo, temp_hi = 0;
  96. u64 mchbar_addr;
  97. int ret = 0;
  98. bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0,0));
  99. if (!bridge_dev) {
  100. DRM_DEBUG("no bridge dev?!\n");
  101. ret = -ENODEV;
  102. goto out;
  103. }
  104. if (IS_I965G(dev))
  105. pci_read_config_dword(bridge_dev, reg + 4, &temp_hi);
  106. pci_read_config_dword(bridge_dev, reg, &temp_lo);
  107. mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
  108. /* If ACPI doesn't have it, assume we need to allocate it ourselves */
  109. if (mchbar_addr &&
  110. pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE)) {
  111. ret = 0;
  112. goto out_put;
  113. }
  114. /* Get some space for it */
  115. ret = pci_bus_alloc_resource(bridge_dev->bus, &dev_priv->mch_res,
  116. MCHBAR_SIZE, MCHBAR_SIZE,
  117. PCIBIOS_MIN_MEM,
  118. 0, pcibios_align_resource,
  119. bridge_dev);
  120. if (ret) {
  121. DRM_DEBUG("failed bus alloc: %d\n", ret);
  122. dev_priv->mch_res.start = 0;
  123. goto out_put;
  124. }
  125. if (IS_I965G(dev))
  126. pci_write_config_dword(bridge_dev, reg + 4,
  127. upper_32_bits(dev_priv->mch_res.start));
  128. pci_write_config_dword(bridge_dev, reg,
  129. lower_32_bits(dev_priv->mch_res.start));
  130. out_put:
  131. pci_dev_put(bridge_dev);
  132. out:
  133. return ret;
  134. }
  135. /* Setup MCHBAR if possible, return true if we should disable it again */
  136. static bool
  137. intel_setup_mchbar(struct drm_device *dev)
  138. {
  139. struct pci_dev *bridge_dev;
  140. int mchbar_reg = IS_I965G(dev) ? MCHBAR_I965 : MCHBAR_I915;
  141. u32 temp;
  142. bool need_disable = false, enabled;
  143. bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0,0));
  144. if (!bridge_dev) {
  145. DRM_DEBUG("no bridge dev?!\n");
  146. goto out;
  147. }
  148. if (IS_I915G(dev) || IS_I915GM(dev)) {
  149. pci_read_config_dword(bridge_dev, DEVEN_REG, &temp);
  150. enabled = !!(temp & DEVEN_MCHBAR_EN);
  151. } else {
  152. pci_read_config_dword(bridge_dev, mchbar_reg, &temp);
  153. enabled = temp & 1;
  154. }
  155. /* If it's already enabled, don't have to do anything */
  156. if (enabled)
  157. goto out_put;
  158. if (intel_alloc_mchbar_resource(dev))
  159. goto out_put;
  160. need_disable = true;
  161. /* Space is allocated or reserved, so enable it. */
  162. if (IS_I915G(dev) || IS_I915GM(dev)) {
  163. pci_write_config_dword(bridge_dev, DEVEN_REG,
  164. temp | DEVEN_MCHBAR_EN);
  165. } else {
  166. pci_read_config_dword(bridge_dev, mchbar_reg, &temp);
  167. pci_write_config_dword(bridge_dev, mchbar_reg, temp | 1);
  168. }
  169. out_put:
  170. pci_dev_put(bridge_dev);
  171. out:
  172. return need_disable;
  173. }
  174. static void
  175. intel_teardown_mchbar(struct drm_device *dev, bool disable)
  176. {
  177. drm_i915_private_t *dev_priv = dev->dev_private;
  178. struct pci_dev *bridge_dev;
  179. int mchbar_reg = IS_I965G(dev) ? MCHBAR_I965 : MCHBAR_I915;
  180. u32 temp;
  181. bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0,0));
  182. if (!bridge_dev) {
  183. DRM_DEBUG("no bridge dev?!\n");
  184. return;
  185. }
  186. if (disable) {
  187. if (IS_I915G(dev) || IS_I915GM(dev)) {
  188. pci_read_config_dword(bridge_dev, DEVEN_REG, &temp);
  189. temp &= ~DEVEN_MCHBAR_EN;
  190. pci_write_config_dword(bridge_dev, DEVEN_REG, temp);
  191. } else {
  192. pci_read_config_dword(bridge_dev, mchbar_reg, &temp);
  193. temp &= ~1;
  194. pci_write_config_dword(bridge_dev, mchbar_reg, temp);
  195. }
  196. }
  197. if (dev_priv->mch_res.start)
  198. release_resource(&dev_priv->mch_res);
  199. }
  200. /**
  201. * Detects bit 6 swizzling of address lookup between IGD access and CPU
  202. * access through main memory.
  203. */
  204. void
  205. i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
  206. {
  207. drm_i915_private_t *dev_priv = dev->dev_private;
  208. uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
  209. uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
  210. bool need_disable;
  211. if (!IS_I9XX(dev)) {
  212. /* As far as we know, the 865 doesn't have these bit 6
  213. * swizzling issues.
  214. */
  215. swizzle_x = I915_BIT_6_SWIZZLE_NONE;
  216. swizzle_y = I915_BIT_6_SWIZZLE_NONE;
  217. } else if (IS_MOBILE(dev)) {
  218. uint32_t dcc;
  219. /* Try to make sure MCHBAR is enabled before poking at it */
  220. need_disable = intel_setup_mchbar(dev);
  221. /* On mobile 9xx chipsets, channel interleave by the CPU is
  222. * determined by DCC. For single-channel, neither the CPU
  223. * nor the GPU do swizzling. For dual channel interleaved,
  224. * the GPU's interleave is bit 9 and 10 for X tiled, and bit
  225. * 9 for Y tiled. The CPU's interleave is independent, and
  226. * can be based on either bit 11 (haven't seen this yet) or
  227. * bit 17 (common).
  228. */
  229. dcc = I915_READ(DCC);
  230. switch (dcc & DCC_ADDRESSING_MODE_MASK) {
  231. case DCC_ADDRESSING_MODE_SINGLE_CHANNEL:
  232. case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC:
  233. swizzle_x = I915_BIT_6_SWIZZLE_NONE;
  234. swizzle_y = I915_BIT_6_SWIZZLE_NONE;
  235. break;
  236. case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED:
  237. if (dcc & DCC_CHANNEL_XOR_DISABLE) {
  238. /* This is the base swizzling by the GPU for
  239. * tiled buffers.
  240. */
  241. swizzle_x = I915_BIT_6_SWIZZLE_9_10;
  242. swizzle_y = I915_BIT_6_SWIZZLE_9;
  243. } else if ((dcc & DCC_CHANNEL_XOR_BIT_17) == 0) {
  244. /* Bit 11 swizzling by the CPU in addition. */
  245. swizzle_x = I915_BIT_6_SWIZZLE_9_10_11;
  246. swizzle_y = I915_BIT_6_SWIZZLE_9_11;
  247. } else {
  248. /* Bit 17 swizzling by the CPU in addition. */
  249. swizzle_x = I915_BIT_6_SWIZZLE_9_10_17;
  250. swizzle_y = I915_BIT_6_SWIZZLE_9_17;
  251. }
  252. break;
  253. }
  254. if (dcc == 0xffffffff) {
  255. DRM_ERROR("Couldn't read from MCHBAR. "
  256. "Disabling tiling.\n");
  257. swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
  258. swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
  259. }
  260. intel_teardown_mchbar(dev, need_disable);
  261. } else {
  262. /* The 965, G33, and newer, have a very flexible memory
  263. * configuration. It will enable dual-channel mode
  264. * (interleaving) on as much memory as it can, and the GPU
  265. * will additionally sometimes enable different bit 6
  266. * swizzling for tiled objects from the CPU.
  267. *
  268. * Here's what I found on the G965:
  269. * slot fill memory size swizzling
  270. * 0A 0B 1A 1B 1-ch 2-ch
  271. * 512 0 0 0 512 0 O
  272. * 512 0 512 0 16 1008 X
  273. * 512 0 0 512 16 1008 X
  274. * 0 512 0 512 16 1008 X
  275. * 1024 1024 1024 0 2048 1024 O
  276. *
  277. * We could probably detect this based on either the DRB
  278. * matching, which was the case for the swizzling required in
  279. * the table above, or from the 1-ch value being less than
  280. * the minimum size of a rank.
  281. */
  282. if (I915_READ16(C0DRB3) != I915_READ16(C1DRB3)) {
  283. swizzle_x = I915_BIT_6_SWIZZLE_NONE;
  284. swizzle_y = I915_BIT_6_SWIZZLE_NONE;
  285. } else {
  286. swizzle_x = I915_BIT_6_SWIZZLE_9_10;
  287. swizzle_y = I915_BIT_6_SWIZZLE_9;
  288. }
  289. }
  290. /* FIXME: check with memory config on IGDNG */
  291. if (IS_IGDNG(dev)) {
  292. DRM_ERROR("disable tiling on IGDNG...\n");
  293. swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
  294. swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
  295. }
  296. dev_priv->mm.bit_6_swizzle_x = swizzle_x;
  297. dev_priv->mm.bit_6_swizzle_y = swizzle_y;
  298. }
  299. /**
  300. * Returns the size of the fence for a tiled object of the given size.
  301. */
  302. static int
  303. i915_get_fence_size(struct drm_device *dev, int size)
  304. {
  305. int i;
  306. int start;
  307. if (IS_I965G(dev)) {
  308. /* The 965 can have fences at any page boundary. */
  309. return ALIGN(size, 4096);
  310. } else {
  311. /* Align the size to a power of two greater than the smallest
  312. * fence size.
  313. */
  314. if (IS_I9XX(dev))
  315. start = 1024 * 1024;
  316. else
  317. start = 512 * 1024;
  318. for (i = start; i < size; i <<= 1)
  319. ;
  320. return i;
  321. }
  322. }
  323. /* Check pitch constriants for all chips & tiling formats */
  324. static bool
  325. i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
  326. {
  327. int tile_width;
  328. /* Linear is always fine */
  329. if (tiling_mode == I915_TILING_NONE)
  330. return true;
  331. if (!IS_I9XX(dev) ||
  332. (tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
  333. tile_width = 128;
  334. else
  335. tile_width = 512;
  336. /* check maximum stride & object size */
  337. if (IS_I965G(dev)) {
  338. /* i965 stores the end address of the gtt mapping in the fence
  339. * reg, so dont bother to check the size */
  340. if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
  341. return false;
  342. } else if (IS_I9XX(dev)) {
  343. uint32_t pitch_val = ffs(stride / tile_width) - 1;
  344. /* XXX: For Y tiling, FENCE_MAX_PITCH_VAL is actually 6 (8KB)
  345. * instead of 4 (2KB) on 945s.
  346. */
  347. if (pitch_val > I915_FENCE_MAX_PITCH_VAL ||
  348. size > (I830_FENCE_MAX_SIZE_VAL << 20))
  349. return false;
  350. } else {
  351. uint32_t pitch_val = ffs(stride / tile_width) - 1;
  352. if (pitch_val > I830_FENCE_MAX_PITCH_VAL ||
  353. size > (I830_FENCE_MAX_SIZE_VAL << 19))
  354. return false;
  355. }
  356. /* 965+ just needs multiples of tile width */
  357. if (IS_I965G(dev)) {
  358. if (stride & (tile_width - 1))
  359. return false;
  360. return true;
  361. }
  362. /* Pre-965 needs power of two tile widths */
  363. if (stride < tile_width)
  364. return false;
  365. if (stride & (stride - 1))
  366. return false;
  367. /* We don't handle the aperture area covered by the fence being bigger
  368. * than the object size.
  369. */
  370. if (i915_get_fence_size(dev, size) != size)
  371. return false;
  372. return true;
  373. }
  374. /**
  375. * Sets the tiling mode of an object, returning the required swizzling of
  376. * bit 6 of addresses in the object.
  377. */
  378. int
  379. i915_gem_set_tiling(struct drm_device *dev, void *data,
  380. struct drm_file *file_priv)
  381. {
  382. struct drm_i915_gem_set_tiling *args = data;
  383. drm_i915_private_t *dev_priv = dev->dev_private;
  384. struct drm_gem_object *obj;
  385. struct drm_i915_gem_object *obj_priv;
  386. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  387. if (obj == NULL)
  388. return -EINVAL;
  389. obj_priv = obj->driver_private;
  390. if (!i915_tiling_ok(dev, args->stride, obj->size, args->tiling_mode)) {
  391. drm_gem_object_unreference(obj);
  392. return -EINVAL;
  393. }
  394. mutex_lock(&dev->struct_mutex);
  395. if (args->tiling_mode == I915_TILING_NONE) {
  396. args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
  397. } else {
  398. if (args->tiling_mode == I915_TILING_X)
  399. args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
  400. else
  401. args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
  402. /* Hide bit 17 swizzling from the user. This prevents old Mesa
  403. * from aborting the application on sw fallbacks to bit 17,
  404. * and we use the pread/pwrite bit17 paths to swizzle for it.
  405. * If there was a user that was relying on the swizzle
  406. * information for drm_intel_bo_map()ed reads/writes this would
  407. * break it, but we don't have any of those.
  408. */
  409. if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
  410. args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
  411. if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
  412. args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
  413. /* If we can't handle the swizzling, make it untiled. */
  414. if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
  415. args->tiling_mode = I915_TILING_NONE;
  416. args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
  417. }
  418. }
  419. if (args->tiling_mode != obj_priv->tiling_mode) {
  420. int ret;
  421. /* Unbind the object, as switching tiling means we're
  422. * switching the cache organization due to fencing, probably.
  423. */
  424. ret = i915_gem_object_unbind(obj);
  425. if (ret != 0) {
  426. WARN(ret != -ERESTARTSYS,
  427. "failed to unbind object for tiling switch");
  428. args->tiling_mode = obj_priv->tiling_mode;
  429. mutex_unlock(&dev->struct_mutex);
  430. drm_gem_object_unreference(obj);
  431. return ret;
  432. }
  433. obj_priv->tiling_mode = args->tiling_mode;
  434. }
  435. obj_priv->stride = args->stride;
  436. drm_gem_object_unreference(obj);
  437. mutex_unlock(&dev->struct_mutex);
  438. return 0;
  439. }
  440. /**
  441. * Returns the current tiling mode and required bit 6 swizzling for the object.
  442. */
  443. int
  444. i915_gem_get_tiling(struct drm_device *dev, void *data,
  445. struct drm_file *file_priv)
  446. {
  447. struct drm_i915_gem_get_tiling *args = data;
  448. drm_i915_private_t *dev_priv = dev->dev_private;
  449. struct drm_gem_object *obj;
  450. struct drm_i915_gem_object *obj_priv;
  451. obj = drm_gem_object_lookup(dev, file_priv, args->handle);
  452. if (obj == NULL)
  453. return -EINVAL;
  454. obj_priv = obj->driver_private;
  455. mutex_lock(&dev->struct_mutex);
  456. args->tiling_mode = obj_priv->tiling_mode;
  457. switch (obj_priv->tiling_mode) {
  458. case I915_TILING_X:
  459. args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
  460. break;
  461. case I915_TILING_Y:
  462. args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
  463. break;
  464. case I915_TILING_NONE:
  465. args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
  466. break;
  467. default:
  468. DRM_ERROR("unknown tiling mode\n");
  469. }
  470. /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
  471. if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
  472. args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
  473. if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
  474. args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
  475. drm_gem_object_unreference(obj);
  476. mutex_unlock(&dev->struct_mutex);
  477. return 0;
  478. }
  479. /**
  480. * Swap every 64 bytes of this page around, to account for it having a new
  481. * bit 17 of its physical address and therefore being interpreted differently
  482. * by the GPU.
  483. */
  484. static int
  485. i915_gem_swizzle_page(struct page *page)
  486. {
  487. char *vaddr;
  488. int i;
  489. char temp[64];
  490. vaddr = kmap(page);
  491. if (vaddr == NULL)
  492. return -ENOMEM;
  493. for (i = 0; i < PAGE_SIZE; i += 128) {
  494. memcpy(temp, &vaddr[i], 64);
  495. memcpy(&vaddr[i], &vaddr[i + 64], 64);
  496. memcpy(&vaddr[i + 64], temp, 64);
  497. }
  498. kunmap(page);
  499. return 0;
  500. }
  501. void
  502. i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj)
  503. {
  504. struct drm_device *dev = obj->dev;
  505. drm_i915_private_t *dev_priv = dev->dev_private;
  506. struct drm_i915_gem_object *obj_priv = obj->driver_private;
  507. int page_count = obj->size >> PAGE_SHIFT;
  508. int i;
  509. if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
  510. return;
  511. if (obj_priv->bit_17 == NULL)
  512. return;
  513. for (i = 0; i < page_count; i++) {
  514. char new_bit_17 = page_to_phys(obj_priv->pages[i]) >> 17;
  515. if ((new_bit_17 & 0x1) !=
  516. (test_bit(i, obj_priv->bit_17) != 0)) {
  517. int ret = i915_gem_swizzle_page(obj_priv->pages[i]);
  518. if (ret != 0) {
  519. DRM_ERROR("Failed to swizzle page\n");
  520. return;
  521. }
  522. set_page_dirty(obj_priv->pages[i]);
  523. }
  524. }
  525. }
  526. void
  527. i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj)
  528. {
  529. struct drm_device *dev = obj->dev;
  530. drm_i915_private_t *dev_priv = dev->dev_private;
  531. struct drm_i915_gem_object *obj_priv = obj->driver_private;
  532. int page_count = obj->size >> PAGE_SHIFT;
  533. int i;
  534. if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
  535. return;
  536. if (obj_priv->bit_17 == NULL) {
  537. obj_priv->bit_17 = kmalloc(BITS_TO_LONGS(page_count) *
  538. sizeof(long), GFP_KERNEL);
  539. if (obj_priv->bit_17 == NULL) {
  540. DRM_ERROR("Failed to allocate memory for bit 17 "
  541. "record\n");
  542. return;
  543. }
  544. }
  545. for (i = 0; i < page_count; i++) {
  546. if (page_to_phys(obj_priv->pages[i]) & (1 << 17))
  547. __set_bit(i, obj_priv->bit_17);
  548. else
  549. __clear_bit(i, obj_priv->bit_17);
  550. }
  551. }