ni.c 44 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532
  1. /*
  2. * Copyright 2010 Advanced Micro Devices, Inc.
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice shall be included in
  12. * all copies or substantial portions of the Software.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  17. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20. * OTHER DEALINGS IN THE SOFTWARE.
  21. *
  22. * Authors: Alex Deucher
  23. */
  24. #include <linux/firmware.h>
  25. #include <linux/platform_device.h>
  26. #include <linux/slab.h>
  27. #include <linux/module.h>
  28. #include "drmP.h"
  29. #include "radeon.h"
  30. #include "radeon_asic.h"
  31. #include "radeon_drm.h"
  32. #include "nid.h"
  33. #include "atom.h"
  34. #include "ni_reg.h"
  35. #include "cayman_blit_shaders.h"
  36. extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  37. extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  38. extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
  39. extern void evergreen_mc_program(struct radeon_device *rdev);
  40. extern void evergreen_irq_suspend(struct radeon_device *rdev);
  41. extern int evergreen_mc_init(struct radeon_device *rdev);
  42. extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
  43. extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
  44. extern void si_rlc_fini(struct radeon_device *rdev);
  45. extern int si_rlc_init(struct radeon_device *rdev);
  46. #define EVERGREEN_PFP_UCODE_SIZE 1120
  47. #define EVERGREEN_PM4_UCODE_SIZE 1376
  48. #define EVERGREEN_RLC_UCODE_SIZE 768
  49. #define BTC_MC_UCODE_SIZE 6024
  50. #define CAYMAN_PFP_UCODE_SIZE 2176
  51. #define CAYMAN_PM4_UCODE_SIZE 2176
  52. #define CAYMAN_RLC_UCODE_SIZE 1024
  53. #define CAYMAN_MC_UCODE_SIZE 6037
  54. #define ARUBA_RLC_UCODE_SIZE 1536
  55. /* Firmware Names */
  56. MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
  57. MODULE_FIRMWARE("radeon/BARTS_me.bin");
  58. MODULE_FIRMWARE("radeon/BARTS_mc.bin");
  59. MODULE_FIRMWARE("radeon/BTC_rlc.bin");
  60. MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
  61. MODULE_FIRMWARE("radeon/TURKS_me.bin");
  62. MODULE_FIRMWARE("radeon/TURKS_mc.bin");
  63. MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
  64. MODULE_FIRMWARE("radeon/CAICOS_me.bin");
  65. MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
  66. MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
  67. MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
  68. MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
  69. MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
  70. MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
  71. MODULE_FIRMWARE("radeon/ARUBA_me.bin");
  72. MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
  73. #define BTC_IO_MC_REGS_SIZE 29
  74. static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
  75. {0x00000077, 0xff010100},
  76. {0x00000078, 0x00000000},
  77. {0x00000079, 0x00001434},
  78. {0x0000007a, 0xcc08ec08},
  79. {0x0000007b, 0x00040000},
  80. {0x0000007c, 0x000080c0},
  81. {0x0000007d, 0x09000000},
  82. {0x0000007e, 0x00210404},
  83. {0x00000081, 0x08a8e800},
  84. {0x00000082, 0x00030444},
  85. {0x00000083, 0x00000000},
  86. {0x00000085, 0x00000001},
  87. {0x00000086, 0x00000002},
  88. {0x00000087, 0x48490000},
  89. {0x00000088, 0x20244647},
  90. {0x00000089, 0x00000005},
  91. {0x0000008b, 0x66030000},
  92. {0x0000008c, 0x00006603},
  93. {0x0000008d, 0x00000100},
  94. {0x0000008f, 0x00001c0a},
  95. {0x00000090, 0xff000001},
  96. {0x00000094, 0x00101101},
  97. {0x00000095, 0x00000fff},
  98. {0x00000096, 0x00116fff},
  99. {0x00000097, 0x60010000},
  100. {0x00000098, 0x10010000},
  101. {0x00000099, 0x00006000},
  102. {0x0000009a, 0x00001000},
  103. {0x0000009f, 0x00946a00}
  104. };
  105. static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
  106. {0x00000077, 0xff010100},
  107. {0x00000078, 0x00000000},
  108. {0x00000079, 0x00001434},
  109. {0x0000007a, 0xcc08ec08},
  110. {0x0000007b, 0x00040000},
  111. {0x0000007c, 0x000080c0},
  112. {0x0000007d, 0x09000000},
  113. {0x0000007e, 0x00210404},
  114. {0x00000081, 0x08a8e800},
  115. {0x00000082, 0x00030444},
  116. {0x00000083, 0x00000000},
  117. {0x00000085, 0x00000001},
  118. {0x00000086, 0x00000002},
  119. {0x00000087, 0x48490000},
  120. {0x00000088, 0x20244647},
  121. {0x00000089, 0x00000005},
  122. {0x0000008b, 0x66030000},
  123. {0x0000008c, 0x00006603},
  124. {0x0000008d, 0x00000100},
  125. {0x0000008f, 0x00001c0a},
  126. {0x00000090, 0xff000001},
  127. {0x00000094, 0x00101101},
  128. {0x00000095, 0x00000fff},
  129. {0x00000096, 0x00116fff},
  130. {0x00000097, 0x60010000},
  131. {0x00000098, 0x10010000},
  132. {0x00000099, 0x00006000},
  133. {0x0000009a, 0x00001000},
  134. {0x0000009f, 0x00936a00}
  135. };
  136. static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
  137. {0x00000077, 0xff010100},
  138. {0x00000078, 0x00000000},
  139. {0x00000079, 0x00001434},
  140. {0x0000007a, 0xcc08ec08},
  141. {0x0000007b, 0x00040000},
  142. {0x0000007c, 0x000080c0},
  143. {0x0000007d, 0x09000000},
  144. {0x0000007e, 0x00210404},
  145. {0x00000081, 0x08a8e800},
  146. {0x00000082, 0x00030444},
  147. {0x00000083, 0x00000000},
  148. {0x00000085, 0x00000001},
  149. {0x00000086, 0x00000002},
  150. {0x00000087, 0x48490000},
  151. {0x00000088, 0x20244647},
  152. {0x00000089, 0x00000005},
  153. {0x0000008b, 0x66030000},
  154. {0x0000008c, 0x00006603},
  155. {0x0000008d, 0x00000100},
  156. {0x0000008f, 0x00001c0a},
  157. {0x00000090, 0xff000001},
  158. {0x00000094, 0x00101101},
  159. {0x00000095, 0x00000fff},
  160. {0x00000096, 0x00116fff},
  161. {0x00000097, 0x60010000},
  162. {0x00000098, 0x10010000},
  163. {0x00000099, 0x00006000},
  164. {0x0000009a, 0x00001000},
  165. {0x0000009f, 0x00916a00}
  166. };
  167. static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
  168. {0x00000077, 0xff010100},
  169. {0x00000078, 0x00000000},
  170. {0x00000079, 0x00001434},
  171. {0x0000007a, 0xcc08ec08},
  172. {0x0000007b, 0x00040000},
  173. {0x0000007c, 0x000080c0},
  174. {0x0000007d, 0x09000000},
  175. {0x0000007e, 0x00210404},
  176. {0x00000081, 0x08a8e800},
  177. {0x00000082, 0x00030444},
  178. {0x00000083, 0x00000000},
  179. {0x00000085, 0x00000001},
  180. {0x00000086, 0x00000002},
  181. {0x00000087, 0x48490000},
  182. {0x00000088, 0x20244647},
  183. {0x00000089, 0x00000005},
  184. {0x0000008b, 0x66030000},
  185. {0x0000008c, 0x00006603},
  186. {0x0000008d, 0x00000100},
  187. {0x0000008f, 0x00001c0a},
  188. {0x00000090, 0xff000001},
  189. {0x00000094, 0x00101101},
  190. {0x00000095, 0x00000fff},
  191. {0x00000096, 0x00116fff},
  192. {0x00000097, 0x60010000},
  193. {0x00000098, 0x10010000},
  194. {0x00000099, 0x00006000},
  195. {0x0000009a, 0x00001000},
  196. {0x0000009f, 0x00976b00}
  197. };
  198. int ni_mc_load_microcode(struct radeon_device *rdev)
  199. {
  200. const __be32 *fw_data;
  201. u32 mem_type, running, blackout = 0;
  202. u32 *io_mc_regs;
  203. int i, ucode_size, regs_size;
  204. if (!rdev->mc_fw)
  205. return -EINVAL;
  206. switch (rdev->family) {
  207. case CHIP_BARTS:
  208. io_mc_regs = (u32 *)&barts_io_mc_regs;
  209. ucode_size = BTC_MC_UCODE_SIZE;
  210. regs_size = BTC_IO_MC_REGS_SIZE;
  211. break;
  212. case CHIP_TURKS:
  213. io_mc_regs = (u32 *)&turks_io_mc_regs;
  214. ucode_size = BTC_MC_UCODE_SIZE;
  215. regs_size = BTC_IO_MC_REGS_SIZE;
  216. break;
  217. case CHIP_CAICOS:
  218. default:
  219. io_mc_regs = (u32 *)&caicos_io_mc_regs;
  220. ucode_size = BTC_MC_UCODE_SIZE;
  221. regs_size = BTC_IO_MC_REGS_SIZE;
  222. break;
  223. case CHIP_CAYMAN:
  224. io_mc_regs = (u32 *)&cayman_io_mc_regs;
  225. ucode_size = CAYMAN_MC_UCODE_SIZE;
  226. regs_size = BTC_IO_MC_REGS_SIZE;
  227. break;
  228. }
  229. mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
  230. running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
  231. if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
  232. if (running) {
  233. blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
  234. WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
  235. }
  236. /* reset the engine and set to writable */
  237. WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
  238. WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
  239. /* load mc io regs */
  240. for (i = 0; i < regs_size; i++) {
  241. WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
  242. WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
  243. }
  244. /* load the MC ucode */
  245. fw_data = (const __be32 *)rdev->mc_fw->data;
  246. for (i = 0; i < ucode_size; i++)
  247. WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
  248. /* put the engine back into the active state */
  249. WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
  250. WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
  251. WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
  252. /* wait for training to complete */
  253. for (i = 0; i < rdev->usec_timeout; i++) {
  254. if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
  255. break;
  256. udelay(1);
  257. }
  258. if (running)
  259. WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
  260. }
  261. return 0;
  262. }
  263. int ni_init_microcode(struct radeon_device *rdev)
  264. {
  265. struct platform_device *pdev;
  266. const char *chip_name;
  267. const char *rlc_chip_name;
  268. size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
  269. char fw_name[30];
  270. int err;
  271. DRM_DEBUG("\n");
  272. pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
  273. err = IS_ERR(pdev);
  274. if (err) {
  275. printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
  276. return -EINVAL;
  277. }
  278. switch (rdev->family) {
  279. case CHIP_BARTS:
  280. chip_name = "BARTS";
  281. rlc_chip_name = "BTC";
  282. pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
  283. me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
  284. rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
  285. mc_req_size = BTC_MC_UCODE_SIZE * 4;
  286. break;
  287. case CHIP_TURKS:
  288. chip_name = "TURKS";
  289. rlc_chip_name = "BTC";
  290. pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
  291. me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
  292. rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
  293. mc_req_size = BTC_MC_UCODE_SIZE * 4;
  294. break;
  295. case CHIP_CAICOS:
  296. chip_name = "CAICOS";
  297. rlc_chip_name = "BTC";
  298. pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
  299. me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
  300. rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
  301. mc_req_size = BTC_MC_UCODE_SIZE * 4;
  302. break;
  303. case CHIP_CAYMAN:
  304. chip_name = "CAYMAN";
  305. rlc_chip_name = "CAYMAN";
  306. pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
  307. me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
  308. rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
  309. mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
  310. break;
  311. case CHIP_ARUBA:
  312. chip_name = "ARUBA";
  313. rlc_chip_name = "ARUBA";
  314. /* pfp/me same size as CAYMAN */
  315. pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
  316. me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
  317. rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
  318. mc_req_size = 0;
  319. break;
  320. default: BUG();
  321. }
  322. DRM_INFO("Loading %s Microcode\n", chip_name);
  323. snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
  324. err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
  325. if (err)
  326. goto out;
  327. if (rdev->pfp_fw->size != pfp_req_size) {
  328. printk(KERN_ERR
  329. "ni_cp: Bogus length %zu in firmware \"%s\"\n",
  330. rdev->pfp_fw->size, fw_name);
  331. err = -EINVAL;
  332. goto out;
  333. }
  334. snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
  335. err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
  336. if (err)
  337. goto out;
  338. if (rdev->me_fw->size != me_req_size) {
  339. printk(KERN_ERR
  340. "ni_cp: Bogus length %zu in firmware \"%s\"\n",
  341. rdev->me_fw->size, fw_name);
  342. err = -EINVAL;
  343. }
  344. snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
  345. err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
  346. if (err)
  347. goto out;
  348. if (rdev->rlc_fw->size != rlc_req_size) {
  349. printk(KERN_ERR
  350. "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
  351. rdev->rlc_fw->size, fw_name);
  352. err = -EINVAL;
  353. }
  354. /* no MC ucode on TN */
  355. if (!(rdev->flags & RADEON_IS_IGP)) {
  356. snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
  357. err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
  358. if (err)
  359. goto out;
  360. if (rdev->mc_fw->size != mc_req_size) {
  361. printk(KERN_ERR
  362. "ni_mc: Bogus length %zu in firmware \"%s\"\n",
  363. rdev->mc_fw->size, fw_name);
  364. err = -EINVAL;
  365. }
  366. }
  367. out:
  368. platform_device_unregister(pdev);
  369. if (err) {
  370. if (err != -EINVAL)
  371. printk(KERN_ERR
  372. "ni_cp: Failed to load firmware \"%s\"\n",
  373. fw_name);
  374. release_firmware(rdev->pfp_fw);
  375. rdev->pfp_fw = NULL;
  376. release_firmware(rdev->me_fw);
  377. rdev->me_fw = NULL;
  378. release_firmware(rdev->rlc_fw);
  379. rdev->rlc_fw = NULL;
  380. release_firmware(rdev->mc_fw);
  381. rdev->mc_fw = NULL;
  382. }
  383. return err;
  384. }
  385. /*
  386. * Core functions
  387. */
  388. static void cayman_gpu_init(struct radeon_device *rdev)
  389. {
  390. u32 gb_addr_config = 0;
  391. u32 mc_shared_chmap, mc_arb_ramcfg;
  392. u32 cgts_tcc_disable;
  393. u32 sx_debug_1;
  394. u32 smx_dc_ctl0;
  395. u32 cgts_sm_ctrl_reg;
  396. u32 hdp_host_path_cntl;
  397. u32 tmp;
  398. u32 disabled_rb_mask;
  399. int i, j;
  400. switch (rdev->family) {
  401. case CHIP_CAYMAN:
  402. rdev->config.cayman.max_shader_engines = 2;
  403. rdev->config.cayman.max_pipes_per_simd = 4;
  404. rdev->config.cayman.max_tile_pipes = 8;
  405. rdev->config.cayman.max_simds_per_se = 12;
  406. rdev->config.cayman.max_backends_per_se = 4;
  407. rdev->config.cayman.max_texture_channel_caches = 8;
  408. rdev->config.cayman.max_gprs = 256;
  409. rdev->config.cayman.max_threads = 256;
  410. rdev->config.cayman.max_gs_threads = 32;
  411. rdev->config.cayman.max_stack_entries = 512;
  412. rdev->config.cayman.sx_num_of_sets = 8;
  413. rdev->config.cayman.sx_max_export_size = 256;
  414. rdev->config.cayman.sx_max_export_pos_size = 64;
  415. rdev->config.cayman.sx_max_export_smx_size = 192;
  416. rdev->config.cayman.max_hw_contexts = 8;
  417. rdev->config.cayman.sq_num_cf_insts = 2;
  418. rdev->config.cayman.sc_prim_fifo_size = 0x100;
  419. rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
  420. rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
  421. gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
  422. break;
  423. case CHIP_ARUBA:
  424. default:
  425. rdev->config.cayman.max_shader_engines = 1;
  426. rdev->config.cayman.max_pipes_per_simd = 4;
  427. rdev->config.cayman.max_tile_pipes = 2;
  428. if ((rdev->pdev->device == 0x9900) ||
  429. (rdev->pdev->device == 0x9901) ||
  430. (rdev->pdev->device == 0x9905) ||
  431. (rdev->pdev->device == 0x9906) ||
  432. (rdev->pdev->device == 0x9907) ||
  433. (rdev->pdev->device == 0x9908) ||
  434. (rdev->pdev->device == 0x9909) ||
  435. (rdev->pdev->device == 0x9910) ||
  436. (rdev->pdev->device == 0x9917)) {
  437. rdev->config.cayman.max_simds_per_se = 6;
  438. rdev->config.cayman.max_backends_per_se = 2;
  439. } else if ((rdev->pdev->device == 0x9903) ||
  440. (rdev->pdev->device == 0x9904) ||
  441. (rdev->pdev->device == 0x990A) ||
  442. (rdev->pdev->device == 0x9913) ||
  443. (rdev->pdev->device == 0x9918)) {
  444. rdev->config.cayman.max_simds_per_se = 4;
  445. rdev->config.cayman.max_backends_per_se = 2;
  446. } else if ((rdev->pdev->device == 0x9919) ||
  447. (rdev->pdev->device == 0x9990) ||
  448. (rdev->pdev->device == 0x9991) ||
  449. (rdev->pdev->device == 0x9994) ||
  450. (rdev->pdev->device == 0x99A0)) {
  451. rdev->config.cayman.max_simds_per_se = 3;
  452. rdev->config.cayman.max_backends_per_se = 1;
  453. } else {
  454. rdev->config.cayman.max_simds_per_se = 2;
  455. rdev->config.cayman.max_backends_per_se = 1;
  456. }
  457. rdev->config.cayman.max_texture_channel_caches = 2;
  458. rdev->config.cayman.max_gprs = 256;
  459. rdev->config.cayman.max_threads = 256;
  460. rdev->config.cayman.max_gs_threads = 32;
  461. rdev->config.cayman.max_stack_entries = 512;
  462. rdev->config.cayman.sx_num_of_sets = 8;
  463. rdev->config.cayman.sx_max_export_size = 256;
  464. rdev->config.cayman.sx_max_export_pos_size = 64;
  465. rdev->config.cayman.sx_max_export_smx_size = 192;
  466. rdev->config.cayman.max_hw_contexts = 8;
  467. rdev->config.cayman.sq_num_cf_insts = 2;
  468. rdev->config.cayman.sc_prim_fifo_size = 0x40;
  469. rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
  470. rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
  471. gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
  472. break;
  473. }
  474. /* Initialize HDP */
  475. for (i = 0, j = 0; i < 32; i++, j += 0x18) {
  476. WREG32((0x2c14 + j), 0x00000000);
  477. WREG32((0x2c18 + j), 0x00000000);
  478. WREG32((0x2c1c + j), 0x00000000);
  479. WREG32((0x2c20 + j), 0x00000000);
  480. WREG32((0x2c24 + j), 0x00000000);
  481. }
  482. WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
  483. evergreen_fix_pci_max_read_req_size(rdev);
  484. mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
  485. mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
  486. tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
  487. rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
  488. if (rdev->config.cayman.mem_row_size_in_kb > 4)
  489. rdev->config.cayman.mem_row_size_in_kb = 4;
  490. /* XXX use MC settings? */
  491. rdev->config.cayman.shader_engine_tile_size = 32;
  492. rdev->config.cayman.num_gpus = 1;
  493. rdev->config.cayman.multi_gpu_tile_size = 64;
  494. tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
  495. rdev->config.cayman.num_tile_pipes = (1 << tmp);
  496. tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
  497. rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
  498. tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
  499. rdev->config.cayman.num_shader_engines = tmp + 1;
  500. tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
  501. rdev->config.cayman.num_gpus = tmp + 1;
  502. tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
  503. rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
  504. tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
  505. rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
  506. /* setup tiling info dword. gb_addr_config is not adequate since it does
  507. * not have bank info, so create a custom tiling dword.
  508. * bits 3:0 num_pipes
  509. * bits 7:4 num_banks
  510. * bits 11:8 group_size
  511. * bits 15:12 row_size
  512. */
  513. rdev->config.cayman.tile_config = 0;
  514. switch (rdev->config.cayman.num_tile_pipes) {
  515. case 1:
  516. default:
  517. rdev->config.cayman.tile_config |= (0 << 0);
  518. break;
  519. case 2:
  520. rdev->config.cayman.tile_config |= (1 << 0);
  521. break;
  522. case 4:
  523. rdev->config.cayman.tile_config |= (2 << 0);
  524. break;
  525. case 8:
  526. rdev->config.cayman.tile_config |= (3 << 0);
  527. break;
  528. }
  529. /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
  530. if (rdev->flags & RADEON_IS_IGP)
  531. rdev->config.cayman.tile_config |= 1 << 4;
  532. else {
  533. if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
  534. rdev->config.cayman.tile_config |= 1 << 4;
  535. else
  536. rdev->config.cayman.tile_config |= 0 << 4;
  537. }
  538. rdev->config.cayman.tile_config |=
  539. ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
  540. rdev->config.cayman.tile_config |=
  541. ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
  542. tmp = 0;
  543. for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
  544. u32 rb_disable_bitmap;
  545. WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
  546. WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
  547. rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
  548. tmp <<= 4;
  549. tmp |= rb_disable_bitmap;
  550. }
  551. /* enabled rb are just the one not disabled :) */
  552. disabled_rb_mask = tmp;
  553. WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
  554. WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
  555. WREG32(GB_ADDR_CONFIG, gb_addr_config);
  556. WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
  557. WREG32(HDP_ADDR_CONFIG, gb_addr_config);
  558. tmp = gb_addr_config & NUM_PIPES_MASK;
  559. tmp = r6xx_remap_render_backend(rdev, tmp,
  560. rdev->config.cayman.max_backends_per_se *
  561. rdev->config.cayman.max_shader_engines,
  562. CAYMAN_MAX_BACKENDS, disabled_rb_mask);
  563. WREG32(GB_BACKEND_MAP, tmp);
  564. cgts_tcc_disable = 0xffff0000;
  565. for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
  566. cgts_tcc_disable &= ~(1 << (16 + i));
  567. WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
  568. WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
  569. WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
  570. WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
  571. /* reprogram the shader complex */
  572. cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
  573. for (i = 0; i < 16; i++)
  574. WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
  575. WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
  576. /* set HW defaults for 3D engine */
  577. WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
  578. sx_debug_1 = RREG32(SX_DEBUG_1);
  579. sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
  580. WREG32(SX_DEBUG_1, sx_debug_1);
  581. smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
  582. smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
  583. smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
  584. WREG32(SMX_DC_CTL0, smx_dc_ctl0);
  585. WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
  586. /* need to be explicitly zero-ed */
  587. WREG32(VGT_OFFCHIP_LDS_BASE, 0);
  588. WREG32(SQ_LSTMP_RING_BASE, 0);
  589. WREG32(SQ_HSTMP_RING_BASE, 0);
  590. WREG32(SQ_ESTMP_RING_BASE, 0);
  591. WREG32(SQ_GSTMP_RING_BASE, 0);
  592. WREG32(SQ_VSTMP_RING_BASE, 0);
  593. WREG32(SQ_PSTMP_RING_BASE, 0);
  594. WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
  595. WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
  596. POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
  597. SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
  598. WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
  599. SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
  600. SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
  601. WREG32(VGT_NUM_INSTANCES, 1);
  602. WREG32(CP_PERFMON_CNTL, 0);
  603. WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
  604. FETCH_FIFO_HIWATER(0x4) |
  605. DONE_FIFO_HIWATER(0xe0) |
  606. ALU_UPDATE_FIFO_HIWATER(0x8)));
  607. WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
  608. WREG32(SQ_CONFIG, (VC_ENABLE |
  609. EXPORT_SRC_C |
  610. GFX_PRIO(0) |
  611. CS1_PRIO(0) |
  612. CS2_PRIO(1)));
  613. WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
  614. WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
  615. FORCE_EOV_MAX_REZ_CNT(255)));
  616. WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
  617. AUTO_INVLD_EN(ES_AND_GS_AUTO));
  618. WREG32(VGT_GS_VERTEX_REUSE, 16);
  619. WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
  620. WREG32(CB_PERF_CTR0_SEL_0, 0);
  621. WREG32(CB_PERF_CTR0_SEL_1, 0);
  622. WREG32(CB_PERF_CTR1_SEL_0, 0);
  623. WREG32(CB_PERF_CTR1_SEL_1, 0);
  624. WREG32(CB_PERF_CTR2_SEL_0, 0);
  625. WREG32(CB_PERF_CTR2_SEL_1, 0);
  626. WREG32(CB_PERF_CTR3_SEL_0, 0);
  627. WREG32(CB_PERF_CTR3_SEL_1, 0);
  628. tmp = RREG32(HDP_MISC_CNTL);
  629. tmp |= HDP_FLUSH_INVALIDATE_CACHE;
  630. WREG32(HDP_MISC_CNTL, tmp);
  631. hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
  632. WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
  633. WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
  634. udelay(50);
  635. }
  636. /*
  637. * GART
  638. */
  639. void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
  640. {
  641. /* flush hdp cache */
  642. WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
  643. /* bits 0-7 are the VM contexts0-7 */
  644. WREG32(VM_INVALIDATE_REQUEST, 1);
  645. }
  646. int cayman_pcie_gart_enable(struct radeon_device *rdev)
  647. {
  648. int i, r;
  649. if (rdev->gart.robj == NULL) {
  650. dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
  651. return -EINVAL;
  652. }
  653. r = radeon_gart_table_vram_pin(rdev);
  654. if (r)
  655. return r;
  656. radeon_gart_restore(rdev);
  657. /* Setup TLB control */
  658. WREG32(MC_VM_MX_L1_TLB_CNTL,
  659. (0xA << 7) |
  660. ENABLE_L1_TLB |
  661. ENABLE_L1_FRAGMENT_PROCESSING |
  662. SYSTEM_ACCESS_MODE_NOT_IN_SYS |
  663. ENABLE_ADVANCED_DRIVER_MODEL |
  664. SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
  665. /* Setup L2 cache */
  666. WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
  667. ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
  668. ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
  669. EFFECTIVE_L2_QUEUE_SIZE(7) |
  670. CONTEXT1_IDENTITY_ACCESS_MODE(1));
  671. WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
  672. WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
  673. L2_CACHE_BIGK_FRAGMENT_SIZE(6));
  674. /* setup context0 */
  675. WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
  676. WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
  677. WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
  678. WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
  679. (u32)(rdev->dummy_page.addr >> 12));
  680. WREG32(VM_CONTEXT0_CNTL2, 0);
  681. WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
  682. RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
  683. WREG32(0x15D4, 0);
  684. WREG32(0x15D8, 0);
  685. WREG32(0x15DC, 0);
  686. /* empty context1-7 */
  687. for (i = 1; i < 8; i++) {
  688. WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
  689. WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), 0);
  690. WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
  691. rdev->gart.table_addr >> 12);
  692. }
  693. /* enable context1-7 */
  694. WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
  695. (u32)(rdev->dummy_page.addr >> 12));
  696. WREG32(VM_CONTEXT1_CNTL2, 0);
  697. WREG32(VM_CONTEXT1_CNTL, 0);
  698. WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
  699. RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
  700. cayman_pcie_gart_tlb_flush(rdev);
  701. DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
  702. (unsigned)(rdev->mc.gtt_size >> 20),
  703. (unsigned long long)rdev->gart.table_addr);
  704. rdev->gart.ready = true;
  705. return 0;
  706. }
  707. void cayman_pcie_gart_disable(struct radeon_device *rdev)
  708. {
  709. /* Disable all tables */
  710. WREG32(VM_CONTEXT0_CNTL, 0);
  711. WREG32(VM_CONTEXT1_CNTL, 0);
  712. /* Setup TLB control */
  713. WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
  714. SYSTEM_ACCESS_MODE_NOT_IN_SYS |
  715. SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
  716. /* Setup L2 cache */
  717. WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
  718. ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
  719. EFFECTIVE_L2_QUEUE_SIZE(7) |
  720. CONTEXT1_IDENTITY_ACCESS_MODE(1));
  721. WREG32(VM_L2_CNTL2, 0);
  722. WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
  723. L2_CACHE_BIGK_FRAGMENT_SIZE(6));
  724. radeon_gart_table_vram_unpin(rdev);
  725. }
  726. void cayman_pcie_gart_fini(struct radeon_device *rdev)
  727. {
  728. cayman_pcie_gart_disable(rdev);
  729. radeon_gart_table_vram_free(rdev);
  730. radeon_gart_fini(rdev);
  731. }
  732. void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
  733. int ring, u32 cp_int_cntl)
  734. {
  735. u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
  736. WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
  737. WREG32(CP_INT_CNTL, cp_int_cntl);
  738. }
  739. /*
  740. * CP.
  741. */
  742. void cayman_fence_ring_emit(struct radeon_device *rdev,
  743. struct radeon_fence *fence)
  744. {
  745. struct radeon_ring *ring = &rdev->ring[fence->ring];
  746. u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
  747. /* flush read cache over gart for this vmid */
  748. radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
  749. radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
  750. radeon_ring_write(ring, 0);
  751. radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
  752. radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
  753. radeon_ring_write(ring, 0xFFFFFFFF);
  754. radeon_ring_write(ring, 0);
  755. radeon_ring_write(ring, 10); /* poll interval */
  756. /* EVENT_WRITE_EOP - flush caches, send int */
  757. radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
  758. radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
  759. radeon_ring_write(ring, addr & 0xffffffff);
  760. radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
  761. radeon_ring_write(ring, fence->seq);
  762. radeon_ring_write(ring, 0);
  763. }
  764. void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
  765. {
  766. struct radeon_ring *ring = &rdev->ring[ib->ring];
  767. /* set to DX10/11 mode */
  768. radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
  769. radeon_ring_write(ring, 1);
  770. radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
  771. radeon_ring_write(ring,
  772. #ifdef __BIG_ENDIAN
  773. (2 << 0) |
  774. #endif
  775. (ib->gpu_addr & 0xFFFFFFFC));
  776. radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
  777. radeon_ring_write(ring, ib->length_dw | (ib->vm_id << 24));
  778. /* flush read cache over gart for this vmid */
  779. radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
  780. radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
  781. radeon_ring_write(ring, ib->vm_id);
  782. radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
  783. radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
  784. radeon_ring_write(ring, 0xFFFFFFFF);
  785. radeon_ring_write(ring, 0);
  786. radeon_ring_write(ring, 10); /* poll interval */
  787. }
  788. static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
  789. {
  790. if (enable)
  791. WREG32(CP_ME_CNTL, 0);
  792. else {
  793. radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
  794. WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
  795. WREG32(SCRATCH_UMSK, 0);
  796. }
  797. }
  798. static int cayman_cp_load_microcode(struct radeon_device *rdev)
  799. {
  800. const __be32 *fw_data;
  801. int i;
  802. if (!rdev->me_fw || !rdev->pfp_fw)
  803. return -EINVAL;
  804. cayman_cp_enable(rdev, false);
  805. fw_data = (const __be32 *)rdev->pfp_fw->data;
  806. WREG32(CP_PFP_UCODE_ADDR, 0);
  807. for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
  808. WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
  809. WREG32(CP_PFP_UCODE_ADDR, 0);
  810. fw_data = (const __be32 *)rdev->me_fw->data;
  811. WREG32(CP_ME_RAM_WADDR, 0);
  812. for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
  813. WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
  814. WREG32(CP_PFP_UCODE_ADDR, 0);
  815. WREG32(CP_ME_RAM_WADDR, 0);
  816. WREG32(CP_ME_RAM_RADDR, 0);
  817. return 0;
  818. }
  819. static int cayman_cp_start(struct radeon_device *rdev)
  820. {
  821. struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
  822. int r, i;
  823. r = radeon_ring_lock(rdev, ring, 7);
  824. if (r) {
  825. DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
  826. return r;
  827. }
  828. radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
  829. radeon_ring_write(ring, 0x1);
  830. radeon_ring_write(ring, 0x0);
  831. radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
  832. radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
  833. radeon_ring_write(ring, 0);
  834. radeon_ring_write(ring, 0);
  835. radeon_ring_unlock_commit(rdev, ring);
  836. cayman_cp_enable(rdev, true);
  837. r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
  838. if (r) {
  839. DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
  840. return r;
  841. }
  842. /* setup clear context state */
  843. radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
  844. radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
  845. for (i = 0; i < cayman_default_size; i++)
  846. radeon_ring_write(ring, cayman_default_state[i]);
  847. radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
  848. radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
  849. /* set clear context state */
  850. radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
  851. radeon_ring_write(ring, 0);
  852. /* SQ_VTX_BASE_VTX_LOC */
  853. radeon_ring_write(ring, 0xc0026f00);
  854. radeon_ring_write(ring, 0x00000000);
  855. radeon_ring_write(ring, 0x00000000);
  856. radeon_ring_write(ring, 0x00000000);
  857. /* Clear consts */
  858. radeon_ring_write(ring, 0xc0036f00);
  859. radeon_ring_write(ring, 0x00000bc4);
  860. radeon_ring_write(ring, 0xffffffff);
  861. radeon_ring_write(ring, 0xffffffff);
  862. radeon_ring_write(ring, 0xffffffff);
  863. radeon_ring_write(ring, 0xc0026900);
  864. radeon_ring_write(ring, 0x00000316);
  865. radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
  866. radeon_ring_write(ring, 0x00000010); /* */
  867. radeon_ring_unlock_commit(rdev, ring);
  868. /* XXX init other rings */
  869. return 0;
  870. }
  871. static void cayman_cp_fini(struct radeon_device *rdev)
  872. {
  873. cayman_cp_enable(rdev, false);
  874. radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
  875. }
  876. int cayman_cp_resume(struct radeon_device *rdev)
  877. {
  878. static const int ridx[] = {
  879. RADEON_RING_TYPE_GFX_INDEX,
  880. CAYMAN_RING_TYPE_CP1_INDEX,
  881. CAYMAN_RING_TYPE_CP2_INDEX
  882. };
  883. static const unsigned cp_rb_cntl[] = {
  884. CP_RB0_CNTL,
  885. CP_RB1_CNTL,
  886. CP_RB2_CNTL,
  887. };
  888. static const unsigned cp_rb_rptr_addr[] = {
  889. CP_RB0_RPTR_ADDR,
  890. CP_RB1_RPTR_ADDR,
  891. CP_RB2_RPTR_ADDR
  892. };
  893. static const unsigned cp_rb_rptr_addr_hi[] = {
  894. CP_RB0_RPTR_ADDR_HI,
  895. CP_RB1_RPTR_ADDR_HI,
  896. CP_RB2_RPTR_ADDR_HI
  897. };
  898. static const unsigned cp_rb_base[] = {
  899. CP_RB0_BASE,
  900. CP_RB1_BASE,
  901. CP_RB2_BASE
  902. };
  903. struct radeon_ring *ring;
  904. int i, r;
  905. /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
  906. WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
  907. SOFT_RESET_PA |
  908. SOFT_RESET_SH |
  909. SOFT_RESET_VGT |
  910. SOFT_RESET_SPI |
  911. SOFT_RESET_SX));
  912. RREG32(GRBM_SOFT_RESET);
  913. mdelay(15);
  914. WREG32(GRBM_SOFT_RESET, 0);
  915. RREG32(GRBM_SOFT_RESET);
  916. WREG32(CP_SEM_WAIT_TIMER, 0x0);
  917. WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
  918. /* Set the write pointer delay */
  919. WREG32(CP_RB_WPTR_DELAY, 0);
  920. WREG32(CP_DEBUG, (1 << 27));
  921. /* set the wb address wether it's enabled or not */
  922. WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
  923. WREG32(SCRATCH_UMSK, 0xff);
  924. for (i = 0; i < 3; ++i) {
  925. uint32_t rb_cntl;
  926. uint64_t addr;
  927. /* Set ring buffer size */
  928. ring = &rdev->ring[ridx[i]];
  929. rb_cntl = drm_order(ring->ring_size / 8);
  930. rb_cntl |= drm_order(RADEON_GPU_PAGE_SIZE/8) << 8;
  931. #ifdef __BIG_ENDIAN
  932. rb_cntl |= BUF_SWAP_32BIT;
  933. #endif
  934. WREG32(cp_rb_cntl[i], rb_cntl);
  935. /* set the wb address wether it's enabled or not */
  936. addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
  937. WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
  938. WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
  939. }
  940. /* set the rb base addr, this causes an internal reset of ALL rings */
  941. for (i = 0; i < 3; ++i) {
  942. ring = &rdev->ring[ridx[i]];
  943. WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
  944. }
  945. for (i = 0; i < 3; ++i) {
  946. /* Initialize the ring buffer's read and write pointers */
  947. ring = &rdev->ring[ridx[i]];
  948. WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
  949. ring->rptr = ring->wptr = 0;
  950. WREG32(ring->rptr_reg, ring->rptr);
  951. WREG32(ring->wptr_reg, ring->wptr);
  952. mdelay(1);
  953. WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
  954. }
  955. /* start the rings */
  956. cayman_cp_start(rdev);
  957. rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
  958. rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
  959. rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
  960. /* this only test cp0 */
  961. r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
  962. if (r) {
  963. rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
  964. rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
  965. rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
  966. return r;
  967. }
  968. return 0;
  969. }
  970. static int cayman_gpu_soft_reset(struct radeon_device *rdev)
  971. {
  972. struct evergreen_mc_save save;
  973. u32 grbm_reset = 0;
  974. if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
  975. return 0;
  976. dev_info(rdev->dev, "GPU softreset \n");
  977. dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
  978. RREG32(GRBM_STATUS));
  979. dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
  980. RREG32(GRBM_STATUS_SE0));
  981. dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
  982. RREG32(GRBM_STATUS_SE1));
  983. dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
  984. RREG32(SRBM_STATUS));
  985. dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_ADDR 0x%08X\n",
  986. RREG32(0x14F8));
  987. dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
  988. RREG32(0x14D8));
  989. dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
  990. RREG32(0x14FC));
  991. dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
  992. RREG32(0x14DC));
  993. evergreen_mc_stop(rdev, &save);
  994. if (evergreen_mc_wait_for_idle(rdev)) {
  995. dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
  996. }
  997. /* Disable CP parsing/prefetching */
  998. WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
  999. /* reset all the gfx blocks */
  1000. grbm_reset = (SOFT_RESET_CP |
  1001. SOFT_RESET_CB |
  1002. SOFT_RESET_DB |
  1003. SOFT_RESET_GDS |
  1004. SOFT_RESET_PA |
  1005. SOFT_RESET_SC |
  1006. SOFT_RESET_SPI |
  1007. SOFT_RESET_SH |
  1008. SOFT_RESET_SX |
  1009. SOFT_RESET_TC |
  1010. SOFT_RESET_TA |
  1011. SOFT_RESET_VGT |
  1012. SOFT_RESET_IA);
  1013. dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
  1014. WREG32(GRBM_SOFT_RESET, grbm_reset);
  1015. (void)RREG32(GRBM_SOFT_RESET);
  1016. udelay(50);
  1017. WREG32(GRBM_SOFT_RESET, 0);
  1018. (void)RREG32(GRBM_SOFT_RESET);
  1019. /* Wait a little for things to settle down */
  1020. udelay(50);
  1021. dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
  1022. RREG32(GRBM_STATUS));
  1023. dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
  1024. RREG32(GRBM_STATUS_SE0));
  1025. dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
  1026. RREG32(GRBM_STATUS_SE1));
  1027. dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
  1028. RREG32(SRBM_STATUS));
  1029. evergreen_mc_resume(rdev, &save);
  1030. return 0;
  1031. }
  1032. int cayman_asic_reset(struct radeon_device *rdev)
  1033. {
  1034. return cayman_gpu_soft_reset(rdev);
  1035. }
  1036. static int cayman_startup(struct radeon_device *rdev)
  1037. {
  1038. struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
  1039. int r;
  1040. /* enable pcie gen2 link */
  1041. evergreen_pcie_gen2_enable(rdev);
  1042. if (rdev->flags & RADEON_IS_IGP) {
  1043. if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
  1044. r = ni_init_microcode(rdev);
  1045. if (r) {
  1046. DRM_ERROR("Failed to load firmware!\n");
  1047. return r;
  1048. }
  1049. }
  1050. } else {
  1051. if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
  1052. r = ni_init_microcode(rdev);
  1053. if (r) {
  1054. DRM_ERROR("Failed to load firmware!\n");
  1055. return r;
  1056. }
  1057. }
  1058. r = ni_mc_load_microcode(rdev);
  1059. if (r) {
  1060. DRM_ERROR("Failed to load MC firmware!\n");
  1061. return r;
  1062. }
  1063. }
  1064. r = r600_vram_scratch_init(rdev);
  1065. if (r)
  1066. return r;
  1067. evergreen_mc_program(rdev);
  1068. r = cayman_pcie_gart_enable(rdev);
  1069. if (r)
  1070. return r;
  1071. cayman_gpu_init(rdev);
  1072. r = evergreen_blit_init(rdev);
  1073. if (r) {
  1074. r600_blit_fini(rdev);
  1075. rdev->asic->copy.copy = NULL;
  1076. dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
  1077. }
  1078. /* allocate rlc buffers */
  1079. if (rdev->flags & RADEON_IS_IGP) {
  1080. r = si_rlc_init(rdev);
  1081. if (r) {
  1082. DRM_ERROR("Failed to init rlc BOs!\n");
  1083. return r;
  1084. }
  1085. }
  1086. /* allocate wb buffer */
  1087. r = radeon_wb_init(rdev);
  1088. if (r)
  1089. return r;
  1090. r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
  1091. if (r) {
  1092. dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
  1093. return r;
  1094. }
  1095. r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
  1096. if (r) {
  1097. dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
  1098. return r;
  1099. }
  1100. r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
  1101. if (r) {
  1102. dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
  1103. return r;
  1104. }
  1105. /* Enable IRQ */
  1106. r = r600_irq_init(rdev);
  1107. if (r) {
  1108. DRM_ERROR("radeon: IH init failed (%d).\n", r);
  1109. radeon_irq_kms_fini(rdev);
  1110. return r;
  1111. }
  1112. evergreen_irq_set(rdev);
  1113. r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
  1114. CP_RB0_RPTR, CP_RB0_WPTR,
  1115. 0, 0xfffff, RADEON_CP_PACKET2);
  1116. if (r)
  1117. return r;
  1118. r = cayman_cp_load_microcode(rdev);
  1119. if (r)
  1120. return r;
  1121. r = cayman_cp_resume(rdev);
  1122. if (r)
  1123. return r;
  1124. r = radeon_ib_pool_init(rdev);
  1125. if (r) {
  1126. dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
  1127. return r;
  1128. }
  1129. r = radeon_ib_ring_tests(rdev);
  1130. if (r)
  1131. return r;
  1132. r = radeon_vm_manager_init(rdev);
  1133. if (r) {
  1134. dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
  1135. return r;
  1136. }
  1137. r = r600_audio_init(rdev);
  1138. if (r)
  1139. return r;
  1140. return 0;
  1141. }
  1142. int cayman_resume(struct radeon_device *rdev)
  1143. {
  1144. int r;
  1145. /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
  1146. * posting will perform necessary task to bring back GPU into good
  1147. * shape.
  1148. */
  1149. /* post card */
  1150. atom_asic_init(rdev->mode_info.atom_context);
  1151. rdev->accel_working = true;
  1152. r = cayman_startup(rdev);
  1153. if (r) {
  1154. DRM_ERROR("cayman startup failed on resume\n");
  1155. rdev->accel_working = false;
  1156. return r;
  1157. }
  1158. return r;
  1159. }
  1160. int cayman_suspend(struct radeon_device *rdev)
  1161. {
  1162. r600_audio_fini(rdev);
  1163. cayman_cp_enable(rdev, false);
  1164. rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
  1165. evergreen_irq_suspend(rdev);
  1166. radeon_wb_disable(rdev);
  1167. cayman_pcie_gart_disable(rdev);
  1168. return 0;
  1169. }
  1170. /* Plan is to move initialization in that function and use
  1171. * helper function so that radeon_device_init pretty much
  1172. * do nothing more than calling asic specific function. This
  1173. * should also allow to remove a bunch of callback function
  1174. * like vram_info.
  1175. */
  1176. int cayman_init(struct radeon_device *rdev)
  1177. {
  1178. struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
  1179. int r;
  1180. /* Read BIOS */
  1181. if (!radeon_get_bios(rdev)) {
  1182. if (ASIC_IS_AVIVO(rdev))
  1183. return -EINVAL;
  1184. }
  1185. /* Must be an ATOMBIOS */
  1186. if (!rdev->is_atom_bios) {
  1187. dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
  1188. return -EINVAL;
  1189. }
  1190. r = radeon_atombios_init(rdev);
  1191. if (r)
  1192. return r;
  1193. /* Post card if necessary */
  1194. if (!radeon_card_posted(rdev)) {
  1195. if (!rdev->bios) {
  1196. dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
  1197. return -EINVAL;
  1198. }
  1199. DRM_INFO("GPU not posted. posting now...\n");
  1200. atom_asic_init(rdev->mode_info.atom_context);
  1201. }
  1202. /* Initialize scratch registers */
  1203. r600_scratch_init(rdev);
  1204. /* Initialize surface registers */
  1205. radeon_surface_init(rdev);
  1206. /* Initialize clocks */
  1207. radeon_get_clock_info(rdev->ddev);
  1208. /* Fence driver */
  1209. r = radeon_fence_driver_init(rdev);
  1210. if (r)
  1211. return r;
  1212. /* initialize memory controller */
  1213. r = evergreen_mc_init(rdev);
  1214. if (r)
  1215. return r;
  1216. /* Memory manager */
  1217. r = radeon_bo_init(rdev);
  1218. if (r)
  1219. return r;
  1220. r = radeon_irq_kms_init(rdev);
  1221. if (r)
  1222. return r;
  1223. ring->ring_obj = NULL;
  1224. r600_ring_init(rdev, ring, 1024 * 1024);
  1225. rdev->ih.ring_obj = NULL;
  1226. r600_ih_ring_init(rdev, 64 * 1024);
  1227. r = r600_pcie_gart_init(rdev);
  1228. if (r)
  1229. return r;
  1230. rdev->accel_working = true;
  1231. r = cayman_startup(rdev);
  1232. if (r) {
  1233. dev_err(rdev->dev, "disabling GPU acceleration\n");
  1234. cayman_cp_fini(rdev);
  1235. r600_irq_fini(rdev);
  1236. if (rdev->flags & RADEON_IS_IGP)
  1237. si_rlc_fini(rdev);
  1238. radeon_wb_fini(rdev);
  1239. radeon_ib_pool_fini(rdev);
  1240. radeon_vm_manager_fini(rdev);
  1241. radeon_irq_kms_fini(rdev);
  1242. cayman_pcie_gart_fini(rdev);
  1243. rdev->accel_working = false;
  1244. }
  1245. /* Don't start up if the MC ucode is missing.
  1246. * The default clocks and voltages before the MC ucode
  1247. * is loaded are not suffient for advanced operations.
  1248. *
  1249. * We can skip this check for TN, because there is no MC
  1250. * ucode.
  1251. */
  1252. if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
  1253. DRM_ERROR("radeon: MC ucode required for NI+.\n");
  1254. return -EINVAL;
  1255. }
  1256. return 0;
  1257. }
  1258. void cayman_fini(struct radeon_device *rdev)
  1259. {
  1260. r600_blit_fini(rdev);
  1261. cayman_cp_fini(rdev);
  1262. r600_irq_fini(rdev);
  1263. if (rdev->flags & RADEON_IS_IGP)
  1264. si_rlc_fini(rdev);
  1265. radeon_wb_fini(rdev);
  1266. radeon_vm_manager_fini(rdev);
  1267. radeon_ib_pool_fini(rdev);
  1268. radeon_irq_kms_fini(rdev);
  1269. cayman_pcie_gart_fini(rdev);
  1270. r600_vram_scratch_fini(rdev);
  1271. radeon_gem_fini(rdev);
  1272. radeon_fence_driver_fini(rdev);
  1273. radeon_bo_fini(rdev);
  1274. radeon_atombios_fini(rdev);
  1275. kfree(rdev->bios);
  1276. rdev->bios = NULL;
  1277. }
  1278. /*
  1279. * vm
  1280. */
  1281. int cayman_vm_init(struct radeon_device *rdev)
  1282. {
  1283. /* number of VMs */
  1284. rdev->vm_manager.nvm = 8;
  1285. /* base offset of vram pages */
  1286. if (rdev->flags & RADEON_IS_IGP) {
  1287. u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
  1288. tmp <<= 22;
  1289. rdev->vm_manager.vram_base_offset = tmp;
  1290. } else
  1291. rdev->vm_manager.vram_base_offset = 0;
  1292. return 0;
  1293. }
  1294. void cayman_vm_fini(struct radeon_device *rdev)
  1295. {
  1296. }
  1297. int cayman_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm, int id)
  1298. {
  1299. WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (id << 2), 0);
  1300. WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (id << 2), vm->last_pfn);
  1301. WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (id << 2), vm->pt_gpu_addr >> 12);
  1302. /* flush hdp cache */
  1303. WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
  1304. /* bits 0-7 are the VM contexts0-7 */
  1305. WREG32(VM_INVALIDATE_REQUEST, 1 << id);
  1306. return 0;
  1307. }
  1308. void cayman_vm_unbind(struct radeon_device *rdev, struct radeon_vm *vm)
  1309. {
  1310. WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (vm->id << 2), 0);
  1311. WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (vm->id << 2), 0);
  1312. WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0);
  1313. /* flush hdp cache */
  1314. WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
  1315. /* bits 0-7 are the VM contexts0-7 */
  1316. WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
  1317. }
  1318. void cayman_vm_tlb_flush(struct radeon_device *rdev, struct radeon_vm *vm)
  1319. {
  1320. if (vm->id == -1)
  1321. return;
  1322. /* flush hdp cache */
  1323. WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
  1324. /* bits 0-7 are the VM contexts0-7 */
  1325. WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
  1326. }
  1327. #define R600_PTE_VALID (1 << 0)
  1328. #define R600_PTE_SYSTEM (1 << 1)
  1329. #define R600_PTE_SNOOPED (1 << 2)
  1330. #define R600_PTE_READABLE (1 << 5)
  1331. #define R600_PTE_WRITEABLE (1 << 6)
  1332. uint32_t cayman_vm_page_flags(struct radeon_device *rdev,
  1333. struct radeon_vm *vm,
  1334. uint32_t flags)
  1335. {
  1336. uint32_t r600_flags = 0;
  1337. r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_PTE_VALID : 0;
  1338. r600_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
  1339. r600_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
  1340. if (flags & RADEON_VM_PAGE_SYSTEM) {
  1341. r600_flags |= R600_PTE_SYSTEM;
  1342. r600_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
  1343. }
  1344. return r600_flags;
  1345. }
  1346. void cayman_vm_set_page(struct radeon_device *rdev, struct radeon_vm *vm,
  1347. unsigned pfn, uint64_t addr, uint32_t flags)
  1348. {
  1349. void __iomem *ptr = (void *)vm->pt;
  1350. addr = addr & 0xFFFFFFFFFFFFF000ULL;
  1351. addr |= flags;
  1352. writeq(addr, ptr + (pfn * 8));
  1353. }